{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# NATURAL LANGUAGE PROCESSING WITH TRANSFORMERS\n",
    "# Α Pos Tagger trained on UD treebank with fine-tuning a BERT model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Native BERT implementation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# First install some extra packages\n",
    "# ! pip install pyconll # for UD treebank reading\n",
    "# ! pip install bert-tensorflow # for using bert model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pyconll, pickle, os, random, datetime, warnings, gc, urllib.request, zipfile, collections\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.utils import shuffle\n",
    "from sklearn.metrics import recall_score, precision_score, classification_report, accuracy_score, confusion_matrix, f1_score\n",
    "from sklearn.model_selection import train_test_split, StratifiedKFold, KFold, GridSearchCV, learning_curve, cross_val_score\n",
    "from sklearn import preprocessing\n",
    "from sklearn.metrics.classification import UndefinedMetricWarning\n",
    "\n",
    "from IPython.display import Image \n",
    "warnings.filterwarnings(action='ignore', category=UndefinedMetricWarning)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_confusion_matrix(f1,\n",
    "                          cm,\n",
    "                          target_names,\n",
    "                          title='Confusion matrix',\n",
    "                          cmap=None,\n",
    "                          normalize=True,\n",
    "                          i=1):\n",
    "    \"\"\"\n",
    "    given a sklearn confusion matrix (cm), make a nice plot\n",
    "\n",
    "    Arguments\n",
    "    ---------\n",
    "    cm:           confusion matrix from sklearn.metrics.confusion_matrix\n",
    "\n",
    "    target_names: given classification classes such as [0, 1, 2]\n",
    "                  the class names, for example: ['high', 'medium', 'low']\n",
    "\n",
    "    title:        the text to display at the top of the matrix\n",
    "\n",
    "    cmap:         the gradient of the values displayed from matplotlib.pyplot.cm\n",
    "                  see http://matplotlib.org/examples/color/colormaps_reference.html\n",
    "                  plt.get_cmap('jet') or plt.cm.Blues\n",
    "\n",
    "    normalize:    If False, plot the raw numbers\n",
    "                  If True, plot the proportions\n",
    "\n",
    "    Usage\n",
    "    -----\n",
    "    plot_confusion_matrix(cm           = cm,                  # confusion matrix created by\n",
    "                                                              # sklearn.metrics.confusion_matrix\n",
    "                          normalize    = True,                # show proportions\n",
    "                          target_names = y_labels_vals,       # list of names of the classes\n",
    "                          title        = best_estimator_name) # title of graph\n",
    "\n",
    "    Citiation\n",
    "    ---------\n",
    "    http://scikit-learn.org/stable/auto_examples/model_selection/plot_confusion_matrix.html\n",
    "\n",
    "    \"\"\"\n",
    "    import matplotlib.pyplot as plt\n",
    "    import numpy as np\n",
    "    import itertools\n",
    "\n",
    "    accuracy = np.trace(cm) / float(np.sum(cm))\n",
    "    misclass = 1 - accuracy\n",
    "    plt.figure(figsize=(10, 6))\n",
    "    if cmap is None:\n",
    "        cmap = plt.get_cmap('Blues')\n",
    "\n",
    "    plt.imshow(cm, interpolation='nearest', cmap=cmap)\n",
    "    plt.title(title)\n",
    "    plt.colorbar()\n",
    "\n",
    "    if target_names is not None:\n",
    "        tick_marks = np.arange(len(target_names))\n",
    "        plt.xticks(tick_marks, target_names, rotation=45)\n",
    "        plt.yticks(tick_marks, target_names)\n",
    "\n",
    "    if normalize:\n",
    "        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]\n",
    "\n",
    "\n",
    "    thresh = cm.max() / 1.5 if normalize else cm.max() / 2\n",
    "    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):\n",
    "        if normalize:\n",
    "            plt.text(j, i, \"{:0.4f}\".format(cm[i, j]),\n",
    "                     horizontalalignment=\"center\",\n",
    "                     color=\"white\" if cm[i, j] > thresh else \"black\")\n",
    "        else:\n",
    "            plt.text(j, i, \"{:,}\".format(cm[i, j]),\n",
    "                     horizontalalignment=\"center\",\n",
    "                     color=\"white\" if cm[i, j] > thresh else \"black\")\n",
    "\n",
    "\n",
    "    plt.tight_layout()\n",
    "    plt.ylabel('True label')\n",
    "    plt.xlabel('Predicted label\\naccuracy={:0.4f}; misclass={:0.4f}; f1-score={:0.4f}'.format(accuracy, misclass, f1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "nbpresent": {
     "id": "75d32121-1074-4cd6-8355-599afeff56db"
    }
   },
   "outputs": [],
   "source": [
    "UD_ENGLISH_TRAIN = 'en_partut-ud-train.conllu'\n",
    "UD_ENGLISH_DEV = 'en_partut-ud-dev.conllu'\n",
    "UD_ENGLISH_TEST = 'en_partut-ud-test.conllu'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "nbpresent": {
     "id": "2eb0b136-1052-4d48-8a66-48b5ebc96ff8"
    }
   },
   "outputs": [],
   "source": [
    "def download_files():\n",
    "    print('Downloading English treebank...')\n",
    "    urllib.request.urlretrieve('http://archive.aueb.gr:8085/files/en_partut-ud-dev.conllu', UD_ENGLISH_DEV)\n",
    "    urllib.request.urlretrieve('http://archive.aueb.gr:8085/files/en_partut-ud-test.conllu', UD_ENGLISH_TEST)\n",
    "    urllib.request.urlretrieve('http://archive.aueb.gr:8085/files/en_partut-ud-train.conllu', UD_ENGLISH_TRAIN)\n",
    "    print('Treebank downloaded.')\n",
    "    print('Downloading pre-trained BERT model...')\n",
    "    urllib.request.urlretrieve('https://storage.googleapis.com/bert_models/2018_10_18/uncased_L-12_H-768_A-12.zip', 'uncased_L-12_H-768_A-12.zip')\n",
    "    print('Pre-trained BERT model downloaded.')\n",
    "    print('Extracting zip file...')\n",
    "    zip_ref = zipfile.ZipFile('uncased_L-12_H-768_A-12.zip', 'r')\n",
    "    zip_ref.extractall('.')\n",
    "    zip_ref.close()\n",
    "    print('Zip file extracted.')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "First we download English treebank and pre-trained BERT model:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "nbpresent": {
     "id": "4e617ca6-4cce-42a0-9396-28b9d704df73"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading English treebank...\n",
      "Treebank downloaded.\n",
      "Downloading pre-trained BERT model...\n",
      "Pre-trained BERT model downloaded.\n",
      "Extracting zip file...\n",
      "Zip file extracted.\n"
     ]
    }
   ],
   "source": [
    "download_files()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "nbpresent": {
     "id": "43c9097b-37be-4f3d-b3d3-4a54a94c1174"
    }
   },
   "source": [
    "# Preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "nbpresent": {
     "id": "016836bb-abb2-4235-ab21-df90a6e1e44e"
    }
   },
   "outputs": [],
   "source": [
    "def read_conllu(path):\n",
    "    data = pyconll.load_from_file(path)\n",
    "    tagged_sentences=[]\n",
    "    t=0\n",
    "    for sentence in data:\n",
    "        tagged_sentence=[]\n",
    "        for token in sentence:\n",
    "            if token.upos and token.form:\n",
    "                t+=1\n",
    "                tagged_sentence.append((token.form.lower(), token.upos))\n",
    "        tagged_sentences.append(tagged_sentence)\n",
    "    return tagged_sentences"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "nbpresent": {
     "id": "11bff856-f685-49ef-a4e6-b5f2a768a9ad"
    }
   },
   "source": [
    "Load train, development and test set in the appropriate tagged format, tuple (word, pos-tag)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "nbpresent": {
     "id": "fc49598b-75d3-45e9-9fa3-a7b604234760"
    }
   },
   "outputs": [],
   "source": [
    "train_sentences = read_conllu(UD_ENGLISH_TRAIN)\n",
    "val_sentences = read_conllu(UD_ENGLISH_DEV)\n",
    "test_sentences = read_conllu(UD_ENGLISH_TEST)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "nbpresent": {
     "id": "3f72d1da-c1d1-4e25-a782-e12a66f1cedf"
    }
   },
   "source": [
    "Print some statistics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "nbpresent": {
     "id": "57ee9c67-7ec6-4054-91bc-a9aa896ba18a"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tagged sentences in train set:  1781\n",
      "Tagged words in train set: 43518\n",
      "========================================\n",
      "Tagged sentences in dev set:  156\n",
      "Tagged words in dev set: 2722\n",
      "========================================\n",
      "Tagged sentences in test set:  153\n",
      "Tagged words in test set: 3408\n",
      "****************************************\n",
      "Total sentences in dataset: 2090\n"
     ]
    }
   ],
   "source": [
    "print(\"Tagged sentences in train set: \", len(train_sentences))\n",
    "print(\"Tagged words in train set:\", len([item for sublist in train_sentences for item in sublist]))\n",
    "print(40*'=')\n",
    "print(\"Tagged sentences in dev set: \", len(val_sentences))\n",
    "print(\"Tagged words in dev set:\", len([item for sublist in val_sentences for item in sublist]))\n",
    "print(40*'=')\n",
    "print(\"Tagged sentences in test set: \", len(test_sentences))\n",
    "print(\"Tagged words in test set:\", len([item for sublist in test_sentences for item in sublist]))\n",
    "print(40*'*')\n",
    "print(\"Total sentences in dataset:\", len(train_sentences)+len(val_sentences)+len(test_sentences))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "nbpresent": {
     "id": "eb97895a-d649-4318-b3ac-d7d6b6088a12"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[('distribution', 'NOUN'), ('of', 'ADP'), ('this', 'DET'), ('license', 'NOUN'), ('does', 'AUX'), ('not', 'PART'), ('create', 'VERB'), ('an', 'DET'), ('attorney', 'NOUN'), ('-', 'PUNCT'), ('client', 'NOUN'), ('relationship', 'NOUN'), ('.', 'PUNCT')]\n"
     ]
    }
   ],
   "source": [
    "print(train_sentences[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "nbpresent": {
     "id": "573e2ec4-2df3-49f2-913d-3d0af2d6dc36"
    }
   },
   "outputs": [],
   "source": [
    "# Some usefull functions\n",
    "def tag_sequence(sentences):\n",
    "    return [[t for w, t in sentence] for sentence in sentences]\n",
    "\n",
    "def text_sequence(sentences):\n",
    "    return [[w for w, t in sentence] for sentence in sentences]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "nbpresent": {
     "id": "81781553-5840-4539-8e31-ebf941647e22"
    }
   },
   "outputs": [],
   "source": [
    "train_text = text_sequence(train_sentences)\n",
    "test_text = text_sequence(test_sentences)\n",
    "val_text = text_sequence(val_sentences)\n",
    "\n",
    "train_label = tag_sequence(train_sentences)\n",
    "test_label= tag_sequence(test_sentences)\n",
    "val_label= tag_sequence(val_sentences)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Max sentence length: 209\n"
     ]
    }
   ],
   "source": [
    "MAX_SEQUENCE_LENGTH=len(max(train_sentences+val_sentences+test_sentences, key=len))\n",
    "print('Max sentence length:',MAX_SEQUENCE_LENGTH)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "([['distribution',\n",
       "   'of',\n",
       "   'this',\n",
       "   'license',\n",
       "   'does',\n",
       "   'not',\n",
       "   'create',\n",
       "   'an',\n",
       "   'attorney',\n",
       "   '-',\n",
       "   'client',\n",
       "   'relationship',\n",
       "   '.']],\n",
       " [['NOUN',\n",
       "   'ADP',\n",
       "   'DET',\n",
       "   'NOUN',\n",
       "   'AUX',\n",
       "   'PART',\n",
       "   'VERB',\n",
       "   'DET',\n",
       "   'NOUN',\n",
       "   'PUNCT',\n",
       "   'NOUN',\n",
       "   'NOUN',\n",
       "   'PUNCT']])"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_text[0:1], train_label[0:1]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Convert our UD dataset to NER conllu format and save in 'data' directory:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "def convert_ner_format(text, label, file):\n",
    "    with open(file, 'w') as f:\n",
    "        words=0\n",
    "        i=0\n",
    "        for zip_i in zip(text, label):\n",
    "            a, b = tuple(zip_i)\n",
    "            for r in range(len(a)):\n",
    "                item = a[r]+' '+b[r]\n",
    "                f.write(\"%s\\n\" % item)\n",
    "                words+=1\n",
    "            f.write(\"\\n\")\n",
    "            i+=1\n",
    "            #if i==3: break\n",
    "    print('Sentences:',i,'Words:',words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sentences: 1781 Words: 43518\n",
      "Sentences: 153 Words: 3408\n",
      "Sentences: 156 Words: 2722\n"
     ]
    }
   ],
   "source": [
    "convert_ner_format(train_text, train_label, 'data/train.txt')\n",
    "convert_ner_format(test_text, test_label, 'data/test.txt')\n",
    "convert_ner_format(val_text, val_label, 'data/dev.txt')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Run BERT fine-tuning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "nbpresent": {
     "id": "62b0877c-64fa-4f42-b18c-2fc0f5989223"
    }
   },
   "outputs": [],
   "source": [
    "MAX_SEQUENCE_LENGTH = MAX_SEQUENCE_LENGTH+2 # +2 for [CLS], [SEP] tokens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "! rm -rf ./output/result_dir/*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "W0521 22:59:06.862257 140480743667456 estimator.py:1924] Estimator's model_fn (<function model_fn_builder.<locals>.model_fn at 0x7fc402d87158>) includes params argument, but params are not passed to Estimator.\n",
      "I0521 22:59:06.863543 140480743667456 estimator.py:201] Using config: {'_model_dir': './output/result_dir', '_tf_random_seed': None, '_save_summary_steps': 100, '_save_checkpoints_steps': 1000, '_save_checkpoints_secs': None, '_session_config': allow_soft_placement: true\n",
      "graph_options {\n",
      "  rewrite_options {\n",
      "    meta_optimizer_iterations: ONE\n",
      "  }\n",
      "}\n",
      ", '_keep_checkpoint_max': 5, '_keep_checkpoint_every_n_hours': 10000, '_log_step_count_steps': None, '_train_distribute': None, '_device_fn': None, '_protocol': None, '_eval_distribute': None, '_experimental_distribute': None, '_service': None, '_cluster_spec': <tensorflow.python.training.server_lib.ClusterSpec object at 0x7fc4044af9b0>, '_task_type': 'worker', '_task_id': 0, '_global_id_in_cluster': 0, '_master': '', '_evaluation_master': '', '_is_chief': True, '_num_ps_replicas': 0, '_num_worker_replicas': 1, '_tpu_config': TPUConfig(iterations_per_loop=1000, num_shards=8, num_cores_per_replica=None, per_host_input_for_training=3, tpu_job_name=None, initial_infeed_sleep_secs=None, input_partition_dims=None), '_cluster': None}\n",
      "I0521 22:59:06.864130 140480743667456 tpu_context.py:202] _TPUContext: eval_on_tpu True\n",
      "W0521 22:59:06.864796 140480743667456 tpu_context.py:204] eval_on_tpu ignored because use_tpu is False.\n",
      "I0521 22:59:06.865655 140480743667456 bert_pos.py:331] Writing example 0 of 1614\n",
      "I0521 22:59:06.866527 140480743667456 bert_pos.py:308] *** Example ***\n",
      "I0521 22:59:06.866910 140480743667456 bert_pos.py:309] guid: train-0\n",
      "I0521 22:59:06.867297 140480743667456 bert_pos.py:311] tokens: distribution of this license does not create an attorney - client relationship .\n",
      "I0521 22:59:06.867608 140480743667456 bert_pos.py:312] input_ids: 101 4353 1997 2023 6105 2515 2025 3443 2019 4905 1011 7396 3276 1012 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "I0521 22:59:06.867933 140480743667456 bert_pos.py:313] input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "I0521 22:59:06.868251 140480743667456 bert_pos.py:314] segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "I0521 22:59:06.868551 140480743667456 bert_pos.py:315] label_ids: 19 8 2 6 8 4 10 16 6 8 13 8 8 13 20 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "I0521 22:59:06.869903 140480743667456 bert_pos.py:308] *** Example ***\n",
      "I0521 22:59:06.870187 140480743667456 bert_pos.py:309] guid: train-1\n",
      "I0521 22:59:06.870449 140480743667456 bert_pos.py:311] tokens: creative commons provides this information on an \" as - is \" basis .\n",
      "I0521 22:59:06.870723 140480743667456 bert_pos.py:312] input_ids: 101 5541 7674 3640 2023 2592 2006 2019 1000 2004 1011 2003 1000 3978 1012 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "I0521 22:59:06.870997 140480743667456 bert_pos.py:313] input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "I0521 22:59:06.871280 140480743667456 bert_pos.py:314] segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "I0521 22:59:06.871549 140480743667456 bert_pos.py:315] label_ids: 19 12 12 16 6 8 2 6 13 2 13 16 13 8 13 20 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "I0521 22:59:06.872968 140480743667456 bert_pos.py:308] *** Example ***\n",
      "I0521 22:59:06.873235 140480743667456 bert_pos.py:309] guid: train-2\n",
      "I0521 22:59:06.873540 140480743667456 bert_pos.py:311] tokens: creative commons makes no warrant ##ies regarding the information provided , and disc ##lai ##ms liability for damages resulting from its use .\n",
      "I0521 22:59:06.873830 140480743667456 bert_pos.py:312] input_ids: 101 5541 7674 3084 2053 10943 3111 4953 1996 2592 3024 1010 1998 5860 19771 5244 14000 2005 12394 4525 2013 2049 2224 1012 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "I0521 22:59:06.874127 140480743667456 bert_pos.py:313] input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "I0521 22:59:06.874428 140480743667456 bert_pos.py:314] segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "I0521 22:59:06.874716 140480743667456 bert_pos.py:315] label_ids: 19 12 12 16 6 8 18 16 6 8 16 13 5 16 18 18 8 2 8 16 2 6 8 13 20 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "WARNING: The TensorFlow contrib module will not be included in TensorFlow 2.0.\n",
      "For more information, please see:\n",
      "  * https://github.com/tensorflow/community/blob/master/rfcs/20180907-contrib-sunset.md\n",
      "  * https://github.com/tensorflow/addons\n",
      "If you depend on functionality not listed there, please file an issue.\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "I0521 22:59:07.861638 140480743667456 bert_pos.py:623] ***** Running training *****\n",
      "I0521 22:59:07.862088 140480743667456 bert_pos.py:624]   Num examples = 1614\n",
      "I0521 22:59:07.862474 140480743667456 bert_pos.py:625]   Batch size = 16\n",
      "I0521 22:59:07.862802 140480743667456 bert_pos.py:626]   Num steps = 403\n",
      "W0521 22:59:07.867192 140480743667456 deprecation.py:323] From /home/dbnet/anaconda3/lib/python3.6/site-packages/tensorflow/python/ops/resource_variable_ops.py:435: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Colocations handled automatically by placer.\n",
      "W0521 22:59:07.886538 140480743667456 deprecation.py:323] From /home/dbnet/stout/BERT-POS/bert_pos.py:363: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.cast instead.\n",
      "I0521 22:59:07.901232 140480743667456 estimator.py:1111] Calling model_fn.\n",
      "I0521 22:59:07.901769 140480743667456 tpu_estimator.py:2532] Running train on CPU\n",
      "I0521 22:59:07.902346 140480743667456 bert_pos.py:458] *** Features ***\n",
      "I0521 22:59:07.902828 140480743667456 bert_pos.py:460]   name = input_ids, shape = (16, 211)\n",
      "I0521 22:59:07.903577 140480743667456 bert_pos.py:460]   name = label_ids, shape = (16, 211)\n",
      "I0521 22:59:07.903898 140480743667456 bert_pos.py:460]   name = mask, shape = (16, 211)\n",
      "I0521 22:59:07.904230 140480743667456 bert_pos.py:460]   name = segment_ids, shape = (16, 211)\n",
      "W0521 22:59:07.940070 140480743667456 deprecation.py:506] From /home/dbnet/anaconda3/lib/python3.6/site-packages/bert/modeling.py:358: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.\n",
      "W0521 22:59:07.950121 140480743667456 deprecation.py:323] From /home/dbnet/anaconda3/lib/python3.6/site-packages/bert/modeling.py:671: dense (from tensorflow.python.layers.core) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use keras.layers.dense instead.\n",
      "W0521 22:59:09.448450 140480743667456 deprecation.py:323] From /home/dbnet/anaconda3/lib/python3.6/site-packages/tensorflow/contrib/crf/python/ops/crf.py:213: dynamic_rnn (from tensorflow.python.ops.rnn) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `keras.layers.RNN(cell)`, which is equivalent to this API\n",
      "I0521 22:59:10.674417 140480743667456 bert_pos.py:489] **** Trainable Variables ****\n",
      "I0521 22:59:10.675065 140480743667456 bert_pos.py:495]   name = bert/embeddings/word_embeddings:0, shape = (30522, 768), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.675383 140480743667456 bert_pos.py:495]   name = bert/embeddings/token_type_embeddings:0, shape = (2, 768), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.675791 140480743667456 bert_pos.py:495]   name = bert/embeddings/position_embeddings:0, shape = (512, 768), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.676090 140480743667456 bert_pos.py:495]   name = bert/embeddings/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.676432 140480743667456 bert_pos.py:495]   name = bert/embeddings/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.676745 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_0/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.677040 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_0/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.677359 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_0/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.677667 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_0/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.677979 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_0/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.678267 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_0/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.678579 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_0/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.678871 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_0/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.679150 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_0/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.679486 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_0/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.679764 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_0/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.680065 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_0/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.681581 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_0/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.681878 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_0/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.682171 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_0/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.682501 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_0/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.682761 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_1/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.683027 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_1/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.683287 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_1/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.684087 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_1/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.684457 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_1/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.684735 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_1/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.685039 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_1/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.685319 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_1/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.685614 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_1/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.685902 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_1/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.686179 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_1/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.686483 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_1/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.686798 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_1/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.687086 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_1/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.688156 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_1/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.688458 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_1/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.688758 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_2/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "I0521 22:59:10.689037 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_2/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.689307 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_2/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.689580 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_2/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.689836 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_2/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.690105 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_2/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.690366 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_2/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.690628 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_2/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.690884 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_2/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.691142 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_2/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.691477 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_2/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.691755 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_2/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.692009 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_2/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.692282 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_2/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.692561 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_2/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.692821 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_2/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.693095 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_3/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.693375 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_3/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.693641 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_3/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.693918 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_3/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.694172 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_3/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.694445 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_3/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.694715 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_3/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.694985 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_3/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.695239 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_3/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.695506 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_3/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.695783 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_3/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.696062 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_3/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.696352 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_3/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.696606 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_3/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.696873 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_3/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.697133 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_3/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.697418 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_4/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.697748 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_4/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.698027 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_4/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.698320 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_4/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.698603 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_4/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.698881 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_4/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.699198 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_4/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.699486 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_4/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.699755 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_4/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.700024 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_4/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.700301 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_4/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.700576 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_4/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.700849 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_4/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.701129 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_4/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.701404 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_4/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.701673 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_4/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.701967 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_5/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.702245 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_5/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.702519 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_5/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.702800 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_5/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.703046 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_5/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.703347 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_5/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.703618 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_5/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.703891 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_5/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.704191 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_5/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "I0521 22:59:10.704452 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_5/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.704741 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_5/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.705023 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_5/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.705284 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_5/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.705552 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_5/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.705822 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_5/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.706099 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_5/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.706387 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_6/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.706655 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_6/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.706922 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_6/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.707204 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_6/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.707527 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_6/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.707792 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_6/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.708065 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_6/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.708371 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_6/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.708642 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_6/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.708920 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_6/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.709201 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_6/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.709474 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_6/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.709747 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_6/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.710029 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_6/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.710318 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_6/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.710574 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_6/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.710900 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_7/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.711178 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_7/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.711453 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_7/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.711744 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_7/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.712011 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_7/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.712296 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_7/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.712580 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_7/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.712874 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_7/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.718987 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_7/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.719311 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_7/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.719639 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_7/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.719937 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_7/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.720223 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_7/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.720503 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_7/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.720780 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_7/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.721093 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_7/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.721393 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_8/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.721661 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_8/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.721953 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_8/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.722231 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_8/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.722519 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_8/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.722787 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_8/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.723077 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_8/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.723338 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_8/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.723620 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_8/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.723897 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_8/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.724130 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_8/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.724391 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_8/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.724646 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_8/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.724919 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_8/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.725178 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_8/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.725429 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_8/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.725696 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_9/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "I0521 22:59:10.725978 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_9/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.726240 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_9/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.726515 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_9/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.726801 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_9/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.727092 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_9/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.727386 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_9/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.727649 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_9/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.727925 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_9/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.728227 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_9/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.728507 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_9/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.728790 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_9/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.729044 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_9/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.729324 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_9/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.729600 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_9/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.729872 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_9/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.730163 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_10/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.730424 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_10/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.730697 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_10/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.730987 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_10/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.731261 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_10/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.731542 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_10/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.731815 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_10/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.732096 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_10/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.732376 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_10/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.732677 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_10/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.732976 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_10/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.733266 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_10/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.733567 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_10/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.733891 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_10/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.734182 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_10/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.734442 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_10/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.734732 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_11/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.734986 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_11/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.735265 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_11/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.735543 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_11/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.735841 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_11/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.736148 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_11/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.736420 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_11/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.736708 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_11/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.736982 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_11/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.737258 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_11/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.737541 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_11/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.737823 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_11/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.738103 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_11/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.738382 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_11/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.738659 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_11/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.738934 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_11/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.739213 140480743667456 bert_pos.py:495]   name = bert/pooler/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.739490 140480743667456 bert_pos.py:495]   name = bert/pooler/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 22:59:10.739767 140480743667456 bert_pos.py:495]   name = dense_37/kernel:0, shape = (768, 21)\n",
      "I0521 22:59:10.740036 140480743667456 bert_pos.py:495]   name = dense_37/bias:0, shape = (21,)\n",
      "I0521 22:59:10.740323 140480743667456 bert_pos.py:495]   name = crf_loss/transition:0, shape = (21, 21)\n",
      "W0521 22:59:10.749474 140480743667456 deprecation.py:323] From /home/dbnet/anaconda3/lib/python3.6/site-packages/tensorflow/python/training/learning_rate_decay_v2.py:321: div (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Deprecated in favor of operator or tf.math.divide.\n",
      "I0521 22:59:15.949142 140480743667456 estimator.py:1113] Done calling model_fn.\n",
      "I0521 22:59:15.950613 140480743667456 basic_session_run_hooks.py:527] Create CheckpointSaverHook.\n",
      "I0521 22:59:18.160250 140480743667456 monitored_session.py:222] Graph was finalized.\n",
      "I0521 22:59:22.298165 140480743667456 session_manager.py:491] Running local_init_op.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "I0521 22:59:22.420811 140480743667456 session_manager.py:493] Done running local_init_op.\n",
      "I0521 22:59:26.711293 140480743667456 basic_session_run_hooks.py:594] Saving checkpoints for 0 into ./output/result_dir/model.ckpt.\n",
      "I0521 23:00:18.262902 140480743667456 tpu_estimator.py:1858] global_step/sec: 2.17923\n",
      "I0521 23:00:18.264348 140480743667456 tpu_estimator.py:1859] examples/sec: 34.8676\n",
      "I0521 23:01:02.011348 140480743667456 tpu_estimator.py:1858] global_step/sec: 2.2858\n",
      "I0521 23:01:02.012377 140480743667456 tpu_estimator.py:1859] examples/sec: 36.5728\n",
      "I0521 23:01:46.407683 140480743667456 tpu_estimator.py:1858] global_step/sec: 2.25244\n",
      "I0521 23:01:46.408595 140480743667456 tpu_estimator.py:1859] examples/sec: 36.039\n",
      "I0521 23:02:31.157800 140480743667456 tpu_estimator.py:1858] global_step/sec: 2.23463\n",
      "I0521 23:02:31.158691 140480743667456 tpu_estimator.py:1859] examples/sec: 35.7541\n",
      "I0521 23:02:32.094064 140480743667456 basic_session_run_hooks.py:594] Saving checkpoints for 403 into ./output/result_dir/model.ckpt.\n",
      "I0521 23:02:33.822635 140480743667456 estimator.py:359] Loss for final step: 1.3998098.\n",
      "I0521 23:02:33.823498 140480743667456 error_handling.py:93] training_loop marked as finished\n",
      "I0521 23:02:33.828803 140480743667456 bert_pos.py:331] Writing example 0 of 143\n",
      "I0521 23:02:33.829778 140480743667456 bert_pos.py:308] *** Example ***\n",
      "I0521 23:02:33.830195 140480743667456 bert_pos.py:309] guid: dev-0\n",
      "I0521 23:02:33.830739 140480743667456 bert_pos.py:311] tokens: creative commons corporation is not a law firm and does not provide legal services .\n",
      "I0521 23:02:33.831076 140480743667456 bert_pos.py:312] input_ids: 101 5541 7674 3840 2003 2025 1037 2375 3813 1998 2515 2025 3073 3423 2578 1012 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "I0521 23:02:33.831440 140480743667456 bert_pos.py:313] input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "I0521 23:02:33.831804 140480743667456 bert_pos.py:314] segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "I0521 23:02:33.832274 140480743667456 bert_pos.py:315] label_ids: 19 12 12 8 4 10 6 8 8 5 4 10 16 1 8 13 20 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "I0521 23:02:33.833755 140480743667456 bert_pos.py:308] *** Example ***\n",
      "I0521 23:02:33.834044 140480743667456 bert_pos.py:309] guid: dev-1\n",
      "I0521 23:02:33.834392 140480743667456 bert_pos.py:311] tokens: the work ( as defined below ) is provided under the terms of this creative commons public license ( \" cc ##pl \" or \" license \" ) .\n",
      "I0521 23:02:33.834699 140480743667456 bert_pos.py:312] input_ids: 101 1996 2147 1006 2004 4225 2917 1007 2003 3024 2104 1996 3408 1997 2023 5541 7674 2270 6105 1006 1000 10507 24759 1000 2030 1000 6105 1000 1007 1012 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "I0521 23:02:33.835019 140480743667456 bert_pos.py:313] input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "I0521 23:02:33.835319 140480743667456 bert_pos.py:314] segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "I0521 23:02:33.835625 140480743667456 bert_pos.py:315] label_ids: 19 6 8 13 2 16 3 13 4 16 2 6 8 2 6 12 12 1 8 13 13 12 18 13 5 13 8 13 13 13 20 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "I0521 23:02:33.836971 140480743667456 bert_pos.py:308] *** Example ***\n",
      "I0521 23:02:33.837270 140480743667456 bert_pos.py:309] guid: dev-2\n",
      "I0521 23:02:33.837551 140480743667456 bert_pos.py:311] tokens: 1 . definitions .\n",
      "I0521 23:02:33.837917 140480743667456 bert_pos.py:312] input_ids: 101 1015 1012 15182 1012 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "I0521 23:02:33.838459 140480743667456 bert_pos.py:313] input_mask: 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "I0521 23:02:33.838778 140480743667456 bert_pos.py:314] segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "I0521 23:02:33.839076 140480743667456 bert_pos.py:315] label_ids: 19 9 13 8 13 20 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "I0521 23:02:33.916278 140480743667456 bert_pos.py:639] ***** Running evaluation *****\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "I0521 23:02:33.916749 140480743667456 bert_pos.py:640]   Num examples = 143\n",
      "I0521 23:02:33.917185 140480743667456 bert_pos.py:641]   Batch size = 8\n",
      "I0521 23:02:33.937536 140480743667456 estimator.py:1111] Calling model_fn.\n",
      "I0521 23:02:33.938038 140480743667456 tpu_estimator.py:2532] Running eval on CPU\n",
      "I0521 23:02:33.938572 140480743667456 bert_pos.py:458] *** Features ***\n",
      "I0521 23:02:33.938973 140480743667456 bert_pos.py:460]   name = input_ids, shape = (?, 211)\n",
      "I0521 23:02:33.939327 140480743667456 bert_pos.py:460]   name = label_ids, shape = (?, 211)\n",
      "I0521 23:02:33.939656 140480743667456 bert_pos.py:460]   name = mask, shape = (?, 211)\n",
      "I0521 23:02:33.939974 140480743667456 bert_pos.py:460]   name = segment_ids, shape = (?, 211)\n",
      "I0521 23:02:36.603968 140480743667456 bert_pos.py:489] **** Trainable Variables ****\n",
      "I0521 23:02:36.604617 140480743667456 bert_pos.py:495]   name = bert/embeddings/word_embeddings:0, shape = (30522, 768), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.605110 140480743667456 bert_pos.py:495]   name = bert/embeddings/token_type_embeddings:0, shape = (2, 768), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.605658 140480743667456 bert_pos.py:495]   name = bert/embeddings/position_embeddings:0, shape = (512, 768), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.605969 140480743667456 bert_pos.py:495]   name = bert/embeddings/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.606266 140480743667456 bert_pos.py:495]   name = bert/embeddings/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.606554 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_0/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.606847 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_0/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.607163 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_0/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.607430 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_0/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.607735 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_0/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.608568 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_0/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.608857 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_0/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.609159 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_0/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.609445 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_0/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.609730 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_0/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.610020 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_0/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.610305 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_0/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.610605 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_0/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.611428 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_0/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.611714 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_0/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.612011 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_0/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.612301 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_1/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.612586 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_1/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.612897 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_1/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.613171 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_1/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.613440 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_1/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.613720 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_1/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.613995 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_1/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.614302 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_1/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.615331 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_1/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.615624 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_1/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.615930 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_1/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.616232 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_1/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.616519 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_1/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.616823 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_1/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.617090 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_1/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.617398 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_1/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.617683 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_2/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.617951 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_2/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.618259 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_2/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.618525 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_2/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.618800 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_2/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.619080 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_2/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.619356 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_2/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.619647 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_2/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.620006 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_2/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.620325 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_2/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.620581 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_2/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.620849 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_2/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.621112 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_2/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "I0521 23:02:36.621374 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_2/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.621640 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_2/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.621915 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_2/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.622236 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_3/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.622500 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_3/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.622769 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_3/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.623026 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_3/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.623292 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_3/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.623575 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_3/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.623846 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_3/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.624125 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_3/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.624411 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_3/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.624692 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_3/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.624979 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_3/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.625285 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_3/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.627895 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_3/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.628244 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_3/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.628519 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_3/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.628814 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_3/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.629103 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_4/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.629385 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_4/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.629675 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_4/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.629957 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_4/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.630241 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_4/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.630564 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_4/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.630849 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_4/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.631127 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_4/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.631416 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_4/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.631693 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_4/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.631992 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_4/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.633291 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_4/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.633595 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_4/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.633881 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_4/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.634185 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_4/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.634472 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_4/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.634752 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_5/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.635040 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_5/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.635317 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_5/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.635596 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_5/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.635878 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_5/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.636187 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_5/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.636489 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_5/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.637555 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_5/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.637858 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_5/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.638140 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_5/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.638420 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_5/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.638716 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_5/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.638993 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_5/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.639271 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_5/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.639555 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_5/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.639832 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_5/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.640124 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_6/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.640404 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_6/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.640684 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_6/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.640984 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_6/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.642120 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_6/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "I0521 23:02:36.642405 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_6/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.642685 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_6/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.642969 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_6/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.643261 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_6/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.643537 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_6/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.643839 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_6/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.644603 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_6/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.644908 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_6/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.645341 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_6/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.645614 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_6/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.645873 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_6/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.646145 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_7/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.646405 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_7/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.646691 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_7/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.646956 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_7/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.647237 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_7/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.647527 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_7/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.647826 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_7/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.648845 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_7/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.649159 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_7/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.649454 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_7/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.649732 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_7/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.650043 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_7/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.650332 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_7/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.650627 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_7/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.650926 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_7/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.651201 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_7/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.651483 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_8/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.651780 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_8/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.652044 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_8/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.652328 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_8/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.652591 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_8/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.652873 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_8/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.653155 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_8/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.653434 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_8/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.653720 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_8/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.653975 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_8/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.654236 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_8/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.654511 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_8/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.654822 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_8/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.655104 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_8/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.655386 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_8/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.655701 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_8/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.655985 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_9/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.656322 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_9/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.656638 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_9/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.656922 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_9/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.657216 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_9/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.657504 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_9/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.657796 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_9/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.658076 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_9/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.660641 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_9/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.660905 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_9/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.661191 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_9/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.661475 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_9/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.661753 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_9/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "I0521 23:02:36.662043 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_9/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.662326 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_9/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.662604 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_9/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.662883 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_10/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.663184 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_10/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.663455 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_10/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.663744 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_10/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.664085 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_10/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.664383 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_10/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.664672 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_10/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.664936 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_10/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.665218 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_10/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.665493 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_10/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.665790 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_10/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.666073 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_10/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.666367 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_10/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.667975 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_10/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.668266 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_10/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.668573 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_10/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.669068 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_11/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.669377 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_11/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.669669 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_11/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.670167 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_11/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.670643 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_11/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.670911 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_11/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.671194 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_11/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.671486 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_11/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.671776 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_11/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.672057 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_11/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.672474 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_11/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.672773 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_11/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.673031 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_11/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.673291 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_11/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.673581 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_11/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.674557 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_11/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.674847 140480743667456 bert_pos.py:495]   name = bert/pooler/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.675128 140480743667456 bert_pos.py:495]   name = bert/pooler/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:36.675399 140480743667456 bert_pos.py:495]   name = dense_37/kernel:0, shape = (768, 21)\n",
      "I0521 23:02:36.675693 140480743667456 bert_pos.py:495]   name = dense_37/bias:0, shape = (21,)\n",
      "I0521 23:02:36.675970 140480743667456 bert_pos.py:495]   name = crf_loss/transition:0, shape = (21, 21)\n",
      "W0521 23:02:36.680280 140480743667456 deprecation.py:323] From /home/dbnet/stout/BERT-POS/metrics.py:82: to_int64 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.cast instead.\n",
      "I0521 23:02:36.728214 140480743667456 estimator.py:1113] Done calling model_fn.\n",
      "W0521 23:02:36.728787 140480743667456 deprecation.py:323] From /home/dbnet/anaconda3/lib/python3.6/site-packages/tensorflow/python/ops/metrics_impl.py:363: to_float (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.cast instead.\n",
      "I0521 23:02:36.742793 140480743667456 evaluation.py:257] Starting evaluation at 2019-05-21T20:02:36Z\n",
      "I0521 23:02:37.086570 140480743667456 monitored_session.py:222] Graph was finalized.\n",
      "W0521 23:02:37.087833 140480743667456 deprecation.py:323] From /home/dbnet/anaconda3/lib/python3.6/site-packages/tensorflow/python/training/saver.py:1266: checkpoint_exists (from tensorflow.python.training.checkpoint_management) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use standard file APIs to check for files with this prefix.\n",
      "I0521 23:02:37.089630 140480743667456 saver.py:1270] Restoring parameters from ./output/result_dir/model.ckpt-403\n",
      "I0521 23:02:37.561605 140480743667456 session_manager.py:491] Running local_init_op.\n",
      "I0521 23:02:37.607593 140480743667456 session_manager.py:493] Done running local_init_op.\n",
      "I0521 23:02:39.855277 140480743667456 evaluation.py:277] Finished evaluation at 2019-05-21-20:02:39\n",
      "I0521 23:02:39.858709 140480743667456 estimator.py:1979] Saving dict for global step 403: confusion_matrix = [[  0.   0.   0.   0.   0.   0.   0.   0.   0.   0.   0.   0.   0.   0.\n",
      "    0.   0.   0.   0.   0.   0.   0.]\n",
      " [  0. 185.   0.   5.   0.   0.   0.   0.  13.   0.   0.   0.   3.   0.\n",
      "    0.   0.   4.   0.   0.   0.   0.]\n",
      " [  0.   0. 293.   2.   0.   0.   0.   0.   0.   0.   1.   0.   0.   0.\n",
      "    1.   0.   0.   0.   0.   0.   0.]\n",
      " [  0.   4.   4.  96.   0.   0.   0.   0.   1.   0.   0.   2.   0.   0.\n",
      "    1.   0.   0.   0.   0.   0.   0.]\n",
      " [  0.   0.   0.   0. 123.   0.   0.   0.   0.   0.   0.   0.   0.   0.\n",
      "    0.   0.   1.   0.   0.   0.   0.]\n",
      " [  0.   0.   0.   0.   0.  88.   0.   0.   0.   0.   0.   0.   0.   0.\n",
      "    0.   0.   0.   0.   0.   0.   0.]\n",
      " [  0.   0.   0.   0.   0.   1. 262.   0.   0.   0.   0.   1.   0.   0.\n",
      "    2.   0.   0.   0.   0.   0.   0.]\n",
      " [  0.   0.   0.   0.   0.   0.   0.   0.   0.   0.   0.   0.   0.   0.\n",
      "    0.   0.   0.   0.   0.   0.   0.]\n",
      " [  0.  12.   0.   0.   0.   0.   0.   0. 537.   0.   0.   0.   9.   0.\n",
      "    0.   0.   8.   0.   2.   0.   0.]\n",
      " [  0.   0.   0.   0.   0.   0.   0.   0.   0.  60.   0.   0.   0.   0.\n",
      "    0.   0.   0.   0.   0.   0.   0.]\n",
      " [  0.   0.   1.   0.   1.   0.   0.   0.   0.   0.  54.   0.   0.   0.\n",
      "    0.   0.   0.   0.   0.   0.   0.]\n",
      " [  0.   0.   0.   1.   0.   0.   2.   0.   0.   1.   0. 146.   0.   0.\n",
      "    3.   0.   0.   0.   0.   0.   0.]\n",
      " [  0.   3.   0.   0.   0.   0.   0.   0.   7.   0.   0.   0.  97.   0.\n",
      "    0.   0.   0.   0.   0.   0.   0.]\n",
      " [  0.   0.   0.   0.   0.   0.   0.   0.   0.   0.   0.   0.   0. 350.\n",
      "    0.   0.   0.   0.   3.   0.   0.]\n",
      " [  0.   0.   2.   2.   0.   0.   2.   0.   0.   0.   0.   2.   0.   0.\n",
      "   33.   0.   0.   0.   0.   0.   0.]\n",
      " [  0.   0.   0.   0.   0.   0.   0.   0.   0.   0.   0.   0.   0.   0.\n",
      "    0.   2.   0.   0.   0.   0.   0.]\n",
      " [  0.   3.   0.   0.   5.   0.   0.   0.   4.   0.   0.   0.   0.   0.\n",
      "    0.   0. 264.   0.   0.   0.   0.]\n",
      " [  0.   0.   0.   0.   0.   0.   2.   0.   3.   0.   0.   0.   1.   0.\n",
      "    0.   0.   0.   7.   0.   0.   0.]\n",
      " [  0.   0.   0.   0.   0.   0.   0.   0.   0.   0.   0.   0.   2.   1.\n",
      "    0.   0.   0.   0. 279.   0.   0.]\n",
      " [  0.   0.   0.   0.   0.   0.   0.   0.   0.   0.   0.   0.   0.   0.\n",
      "    0.   0.   0.   0.   0. 143.   0.]\n",
      " [  0.   0.   0.   0.   0.   0.   0.   0.   0.   0.   0.   0.   0.   0.\n",
      "    0.   0.   0.   0.   0.   0. 143.]], global_step = 403, loss = 4.048247\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "I0521 23:02:40.295973 140480743667456 estimator.py:2012] Summary for np.ndarray is not visible in Tensorboard by default. Consider using a Tensorboard plugin for visualization (see https://github.com/tensorflow/tensorboard-plugin-example/blob/master/README.md for more information).\n",
      "I0521 23:02:40.296931 140480743667456 estimator.py:2039] Saving 'checkpoint_path' summary for global step 403: ./output/result_dir/model.ckpt-403\n",
      "I0521 23:02:40.297545 140480743667456 error_handling.py:93] evaluation_loop marked as finished\n",
      "I0521 23:02:40.298076 140480743667456 bert_pos.py:653] ***** Eval results *****\n",
      "I0521 23:02:40.298760 140480743667456 bert_pos.py:656] ***********************************************\n",
      "I0521 23:02:40.299173 140480743667456 bert_pos.py:657] ********************P = 0.8601175613195139*********************\n",
      "I0521 23:02:40.299687 140480743667456 bert_pos.py:658] ********************R = 0.8392247131387066*********************\n",
      "I0521 23:02:40.299961 140480743667456 bert_pos.py:659] ********************F = 0.8461406437605048*********************\n",
      "I0521 23:02:40.300234 140480743667456 bert_pos.py:660] ***********************************************\n",
      "I0521 23:02:40.300649 140480743667456 bert_pos.py:663] ***** Eval results *****\n",
      "I0521 23:02:40.303442 140480743667456 bert_pos.py:665]   confusion_matrix = [[  0.   0.   0.   0.   0.   0.   0.   0.   0.   0.   0.   0.   0.   0.\n",
      "    0.   0.   0.   0.   0.   0.   0.]\n",
      " [  0. 185.   0.   5.   0.   0.   0.   0.  13.   0.   0.   0.   3.   0.\n",
      "    0.   0.   4.   0.   0.   0.   0.]\n",
      " [  0.   0. 293.   2.   0.   0.   0.   0.   0.   0.   1.   0.   0.   0.\n",
      "    1.   0.   0.   0.   0.   0.   0.]\n",
      " [  0.   4.   4.  96.   0.   0.   0.   0.   1.   0.   0.   2.   0.   0.\n",
      "    1.   0.   0.   0.   0.   0.   0.]\n",
      " [  0.   0.   0.   0. 123.   0.   0.   0.   0.   0.   0.   0.   0.   0.\n",
      "    0.   0.   1.   0.   0.   0.   0.]\n",
      " [  0.   0.   0.   0.   0.  88.   0.   0.   0.   0.   0.   0.   0.   0.\n",
      "    0.   0.   0.   0.   0.   0.   0.]\n",
      " [  0.   0.   0.   0.   0.   1. 262.   0.   0.   0.   0.   1.   0.   0.\n",
      "    2.   0.   0.   0.   0.   0.   0.]\n",
      " [  0.   0.   0.   0.   0.   0.   0.   0.   0.   0.   0.   0.   0.   0.\n",
      "    0.   0.   0.   0.   0.   0.   0.]\n",
      " [  0.  12.   0.   0.   0.   0.   0.   0. 537.   0.   0.   0.   9.   0.\n",
      "    0.   0.   8.   0.   2.   0.   0.]\n",
      " [  0.   0.   0.   0.   0.   0.   0.   0.   0.  60.   0.   0.   0.   0.\n",
      "    0.   0.   0.   0.   0.   0.   0.]\n",
      " [  0.   0.   1.   0.   1.   0.   0.   0.   0.   0.  54.   0.   0.   0.\n",
      "    0.   0.   0.   0.   0.   0.   0.]\n",
      " [  0.   0.   0.   1.   0.   0.   2.   0.   0.   1.   0. 146.   0.   0.\n",
      "    3.   0.   0.   0.   0.   0.   0.]\n",
      " [  0.   3.   0.   0.   0.   0.   0.   0.   7.   0.   0.   0.  97.   0.\n",
      "    0.   0.   0.   0.   0.   0.   0.]\n",
      " [  0.   0.   0.   0.   0.   0.   0.   0.   0.   0.   0.   0.   0. 350.\n",
      "    0.   0.   0.   0.   3.   0.   0.]\n",
      " [  0.   0.   2.   2.   0.   0.   2.   0.   0.   0.   0.   2.   0.   0.\n",
      "   33.   0.   0.   0.   0.   0.   0.]\n",
      " [  0.   0.   0.   0.   0.   0.   0.   0.   0.   0.   0.   0.   0.   0.\n",
      "    0.   2.   0.   0.   0.   0.   0.]\n",
      " [  0.   3.   0.   0.   5.   0.   0.   0.   4.   0.   0.   0.   0.   0.\n",
      "    0.   0. 264.   0.   0.   0.   0.]\n",
      " [  0.   0.   0.   0.   0.   0.   2.   0.   3.   0.   0.   0.   1.   0.\n",
      "    0.   0.   0.   7.   0.   0.   0.]\n",
      " [  0.   0.   0.   0.   0.   0.   0.   0.   0.   0.   0.   0.   2.   1.\n",
      "    0.   0.   0.   0. 279.   0.   0.]\n",
      " [  0.   0.   0.   0.   0.   0.   0.   0.   0.   0.   0.   0.   0.   0.\n",
      "    0.   0.   0.   0.   0. 143.   0.]\n",
      " [  0.   0.   0.   0.   0.   0.   0.   0.   0.   0.   0.   0.   0.   0.\n",
      "    0.   0.   0.   0.   0.   0. 143.]]\n",
      "I0521 23:02:40.306099 140480743667456 bert_pos.py:665]   global_step = 403\n",
      "I0521 23:02:40.306442 140480743667456 bert_pos.py:665]   loss = 4.048247\n",
      "I0521 23:02:40.449908 140480743667456 bert_pos.py:331] Writing example 0 of 150\n",
      "I0521 23:02:40.450906 140480743667456 bert_pos.py:308] *** Example ***\n",
      "I0521 23:02:40.451276 140480743667456 bert_pos.py:309] guid: test-0\n",
      "I0521 23:02:40.451548 140480743667456 bert_pos.py:311] tokens: at ##tri ##bution - share ##ali ##ke 2 . 0 .\n",
      "I0521 23:02:40.451844 140480743667456 bert_pos.py:312] input_ids: 101 2012 18886 29446 1011 3745 11475 3489 1016 1012 1014 1012 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "I0521 23:02:40.452404 140480743667456 bert_pos.py:313] input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "I0521 23:02:40.452696 140480743667456 bert_pos.py:314] segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "I0521 23:02:40.452988 140480743667456 bert_pos.py:315] label_ids: 19 8 18 18 13 12 18 18 9 18 18 13 20 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "I0521 23:02:40.823912 140480743667456 bert_pos.py:308] *** Example ***\n",
      "I0521 23:02:40.825608 140480743667456 bert_pos.py:309] guid: test-1\n",
      "I0521 23:02:40.827191 140480743667456 bert_pos.py:311] tokens: any use of the work other than as authorized under this license or copyright law is prohibited .\n",
      "I0521 23:02:40.828873 140480743667456 bert_pos.py:312] input_ids: 101 2151 2224 1997 1996 2147 2060 2084 2004 9362 2104 2023 6105 2030 9385 2375 2003 10890 1012 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "I0521 23:02:40.831748 140480743667456 bert_pos.py:313] input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "I0521 23:02:40.834311 140480743667456 bert_pos.py:314] segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "I0521 23:02:40.836087 140480743667456 bert_pos.py:315] label_ids: 19 6 8 2 6 8 1 14 2 16 2 6 8 5 8 8 4 16 13 20 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "I0521 23:02:41.113238 140480743667456 bert_pos.py:308] *** Example ***\n",
      "I0521 23:02:41.114897 140480743667456 bert_pos.py:309] guid: test-2\n",
      "I0521 23:02:41.116538 140480743667456 bert_pos.py:311] tokens: unless otherwise agreed to by the parties in writing , li ##cens ##or offers the work as - is and makes no representations or warrant ##ies of any kind concerning the materials , express , implied , statutory or otherwise , including , without limitation , warrant ##ies of title , merchant ##ibility , fitness for a particular purpose , non ##in ##fr ##inge ##ment , or the absence of late ##nt or other defects , accuracy , or the presence of absence of errors , whether or not discover ##able .\n",
      "I0521 23:02:41.119356 140480743667456 bert_pos.py:312] input_ids: 101 4983 4728 3530 2000 2011 1996 4243 1999 3015 1010 5622 19023 2953 4107 1996 2147 2004 1011 2003 1998 3084 2053 15066 2030 10943 3111 1997 2151 2785 7175 1996 4475 1010 4671 1010 13339 1010 15201 2030 4728 1010 2164 1010 2302 22718 1010 10943 3111 1997 2516 1010 6432 13464 1010 10516 2005 1037 3327 3800 1010 2512 2378 19699 23496 3672 1010 2030 1996 6438 1997 2397 3372 2030 2060 18419 1010 10640 1010 2030 1996 3739 1997 6438 1997 10697 1010 3251 2030 2025 7523 3085 1012 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "I0521 23:02:41.120602 140480743667456 bert_pos.py:313] input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "I0521 23:02:41.120937 140480743667456 bert_pos.py:314] segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "I0521 23:02:41.121383 140480743667456 bert_pos.py:315] label_ids: 19 14 3 16 2 2 6 8 2 8 13 8 18 18 16 6 8 2 13 16 5 16 6 8 5 8 18 2 6 8 16 6 8 13 1 13 16 13 1 5 3 13 16 13 2 8 13 8 18 2 8 13 8 18 13 8 2 6 1 8 13 8 18 18 18 18 13 5 6 8 2 1 18 5 1 8 13 8 13 5 6 8 2 8 2 8 13 14 5 10 1 18 13 20 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "I0521 23:02:41.207018 140480743667456 bert_pos.py:682] ***** Running prediction*****\n",
      "I0521 23:02:41.207536 140480743667456 bert_pos.py:683]   Num examples = 150\n",
      "I0521 23:02:41.207957 140480743667456 bert_pos.py:684]   Batch size = 8\n",
      "I0521 23:02:41.228329 140480743667456 estimator.py:1111] Calling model_fn.\n",
      "I0521 23:02:41.228897 140480743667456 tpu_estimator.py:2532] Running infer on CPU\n",
      "I0521 23:02:41.229382 140480743667456 bert_pos.py:458] *** Features ***\n",
      "I0521 23:02:41.229777 140480743667456 bert_pos.py:460]   name = input_ids, shape = (?, 211)\n",
      "I0521 23:02:41.230157 140480743667456 bert_pos.py:460]   name = label_ids, shape = (?, 211)\n",
      "I0521 23:02:41.230470 140480743667456 bert_pos.py:460]   name = mask, shape = (?, 211)\n",
      "I0521 23:02:41.230794 140480743667456 bert_pos.py:460]   name = segment_ids, shape = (?, 211)\n",
      "I0521 23:02:43.989258 140480743667456 bert_pos.py:489] **** Trainable Variables ****\n",
      "I0521 23:02:43.989868 140480743667456 bert_pos.py:495]   name = bert/embeddings/word_embeddings:0, shape = (30522, 768), *INIT_FROM_CKPT*\n",
      "I0521 23:02:43.990486 140480743667456 bert_pos.py:495]   name = bert/embeddings/token_type_embeddings:0, shape = (2, 768), *INIT_FROM_CKPT*\n",
      "I0521 23:02:43.990849 140480743667456 bert_pos.py:495]   name = bert/embeddings/position_embeddings:0, shape = (512, 768), *INIT_FROM_CKPT*\n",
      "I0521 23:02:43.991196 140480743667456 bert_pos.py:495]   name = bert/embeddings/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:43.991554 140480743667456 bert_pos.py:495]   name = bert/embeddings/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:43.991899 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_0/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "I0521 23:02:43.992238 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_0/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:43.992536 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_0/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "I0521 23:02:43.993062 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_0/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:43.993835 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_0/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "I0521 23:02:43.994119 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_0/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:43.994420 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_0/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "I0521 23:02:43.994709 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_0/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:43.995291 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_0/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:43.995588 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_0/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:43.996025 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_0/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n",
      "I0521 23:02:43.996325 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_0/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:43.996595 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_0/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n",
      "I0521 23:02:43.996910 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_0/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:43.997191 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_0/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:43.997473 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_0/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:43.997778 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_1/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "I0521 23:02:43.998054 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_1/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:43.998359 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_1/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "I0521 23:02:43.998654 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_1/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:43.998950 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_1/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "I0521 23:02:43.999240 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_1/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:43.999526 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_1/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "I0521 23:02:43.999824 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_1/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.000097 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_1/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "I0521 23:02:44.000418 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_1/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.000691 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_1/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.000997 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_1/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.001280 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_1/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.001553 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_1/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.001839 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_1/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.002105 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_1/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.002367 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_2/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.002632 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_2/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.002921 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_2/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.004901 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_2/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.005216 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_2/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.005512 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_2/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.005967 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_2/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.006350 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_2/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.006640 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_2/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.006901 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_2/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.007162 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_2/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.007422 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_2/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.007683 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_2/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.007946 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_2/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.008219 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_2/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.008497 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_2/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.008780 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_3/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.009074 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_3/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.009352 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_3/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.009618 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_3/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.009892 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_3/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.010170 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_3/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.010446 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_3/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.010748 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_3/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.010997 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_3/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.011257 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_3/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.011537 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_3/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.011797 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_3/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.012049 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_3/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.012351 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_3/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.012648 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_3/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.014688 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_3/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.014973 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_4/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.015268 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_4/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.015525 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_4/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.015822 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_4/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.016072 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_4/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.016331 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_4/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.016594 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_4/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.016847 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_4/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.017106 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_4/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.017369 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_4/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.018371 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_4/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.018641 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_4/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.018890 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_4/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.019153 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_4/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.019406 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_4/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.019669 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_4/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.019930 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_5/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "I0521 23:02:44.020193 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_5/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.020454 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_5/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.020725 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_5/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.020979 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_5/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.021231 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_5/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.021482 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_5/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.021746 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_5/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.022003 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_5/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.022257 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_5/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.022506 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_5/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.022776 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_5/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.023030 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_5/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.023280 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_5/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.023527 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_5/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.023783 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_5/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.024037 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_6/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.024298 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_6/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.024548 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_6/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.024810 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_6/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.025062 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_6/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.025316 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_6/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.025571 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_6/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.025822 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_6/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.026078 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_6/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.026339 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_6/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.026610 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_6/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.026885 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_6/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.027148 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_6/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.027409 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_6/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.027682 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_6/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.027958 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_6/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.028242 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_7/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.028533 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_7/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.028794 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_7/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.029066 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_7/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.029334 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_7/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.029606 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_7/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.029882 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_7/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.030153 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_7/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.030426 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_7/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.030706 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_7/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.030997 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_7/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.031271 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_7/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.031547 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_7/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.031821 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_7/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.032099 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_7/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.032407 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_7/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.032678 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_8/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.032968 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_8/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.033235 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_8/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.033513 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_8/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.033787 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_8/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.034089 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_8/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.034380 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_8/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.034638 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_8/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.034893 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_8/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "I0521 23:02:44.035148 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_8/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.035406 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_8/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.035679 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_8/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.035950 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_8/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.036232 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_8/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.036504 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_8/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.036773 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_8/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.037050 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_9/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.037326 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_9/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.037600 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_9/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.037882 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_9/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.038169 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_9/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.038460 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_9/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.038722 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_9/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.044078 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_9/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.044392 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_9/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.044690 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_9/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.044971 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_9/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.045289 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_9/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.045599 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_9/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.045887 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_9/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.046193 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_9/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.046469 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_9/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.046809 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_10/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.047094 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_10/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.047384 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_10/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.047685 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_10/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.047982 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_10/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.048286 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_10/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.048564 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_10/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.048861 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_10/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.049153 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_10/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.049432 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_10/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.049716 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_10/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.049997 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_10/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.050338 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_10/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.052227 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_10/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.052513 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_10/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.052789 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_10/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.053090 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_11/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.053374 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_11/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.053647 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_11/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.053955 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_11/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.054232 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_11/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.054535 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_11/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.054826 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_11/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.055691 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_11/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.056012 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_11/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.056297 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_11/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.056596 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_11/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.057183 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_11/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.057450 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_11/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.057733 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_11/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.058010 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_11/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.058316 140480743667456 bert_pos.py:495]   name = bert/encoder/layer_11/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.058937 140480743667456 bert_pos.py:495]   name = bert/pooler/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "I0521 23:02:44.059241 140480743667456 bert_pos.py:495]   name = bert/pooler/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I0521 23:02:44.059664 140480743667456 bert_pos.py:495]   name = dense_37/kernel:0, shape = (768, 21)\n",
      "I0521 23:02:44.059941 140480743667456 bert_pos.py:495]   name = dense_37/bias:0, shape = (21,)\n",
      "I0521 23:02:44.060263 140480743667456 bert_pos.py:495]   name = crf_loss/transition:0, shape = (21, 21)\n",
      "I0521 23:02:44.060698 140480743667456 estimator.py:1113] Done calling model_fn.\n",
      "I0521 23:02:44.406044 140480743667456 monitored_session.py:222] Graph was finalized.\n",
      "I0521 23:02:44.408295 140480743667456 saver.py:1270] Restoring parameters from ./output/result_dir/model.ckpt-403\n",
      "I0521 23:02:44.883916 140480743667456 session_manager.py:491] Running local_init_op.\n",
      "I0521 23:02:44.927063 140480743667456 session_manager.py:493] Done running local_init_op.\n",
      "I0521 23:02:47.660350 140480743667456 error_handling.py:93] prediction_loop marked as finished\n",
      "I0521 23:02:47.661215 140480743667456 error_handling.py:93] prediction_loop marked as finished\n"
     ]
    }
   ],
   "source": [
    "%run bert_pos.py   \\\n",
    "    --task_name=\"POS\"  \\\n",
    "    --do_lower_case=False \\ # Our dataset is already lower cased!\n",
    "    --crf=False \\\n",
    "    --do_train=True   \\\n",
    "    --do_eval=True   \\\n",
    "    --do_predict=True \\\n",
    "    --data_dir=data   \\\n",
    "    --vocab_file=uncased_L-12_H-768_A-12/vocab.txt  \\\n",
    "    --bert_config_file=uncased_L-12_H-768_A-12/bert_config.json \\\n",
    "    --init_checkpoint=uncased_L-12_H-768_A-12/bert_model.ckpt   \\\n",
    "    --max_seq_length=$MAX_SEQUENCE_LENGTH  \\\n",
    "    --train_batch_size=16   \\\n",
    "    --learning_rate=2e-5   \\\n",
    "    --num_train_epochs=4.0   \\\n",
    "    --output_dir=./output/result_dir"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Evaluation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Classification report"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Results are stored in *'./output/result_dir/'*. Lets see the classification report:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "predictions ='./output/result_dir/label_test.txt'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_true = []\n",
    "y_pred = []\n",
    "with open(predictions) as fp:\n",
    "    for line in fp:\n",
    "        line = line.split()\n",
    "        if not line[0].startswith('##') and not line[1].startswith('##') :#and not line[2].startswith('##'):\n",
    "            y_true.append(line[1])\n",
    "            y_pred.append(line[2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(3408, 3408)"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(y_true), len(y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "------------ Result of Bert fine-tuned model ----------\n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "          ##     0.0000    0.0000    0.0000         0\n",
      "         ADJ     0.9217    0.8929    0.9070       224\n",
      "         ADP     0.9817    0.9877    0.9847       488\n",
      "         ADV     0.9302    0.9160    0.9231       131\n",
      "         AUX     0.9957    1.0000    0.9979       234\n",
      "       CCONJ     0.9896    0.9896    0.9896        96\n",
      "         DET     0.9955    1.0000    0.9977       439\n",
      "        INTJ     1.0000    1.0000    1.0000         2\n",
      "        NOUN     0.9736    0.9788    0.9762       753\n",
      "         NUM     0.9385    1.0000    0.9683        61\n",
      "        PART     0.9565    1.0000    0.9778        66\n",
      "        PRON     1.0000    0.9340    0.9659       106\n",
      "       PROPN     0.9405    0.8778    0.9080        90\n",
      "       PUNCT     1.0000    0.9971    0.9985       339\n",
      "       SCONJ     0.9565    0.8627    0.9072        51\n",
      "        VERB     0.9580    0.9785    0.9681       326\n",
      "           X     1.0000    1.0000    1.0000         2\n",
      "\n",
      "   micro avg     0.9733    0.9733    0.9733      3408\n",
      "   macro avg     0.9140    0.9068    0.9100      3408\n",
      "weighted avg     0.9743    0.9733    0.9737      3408\n",
      "\n"
     ]
    }
   ],
   "source": [
    "name='Bert fine-tuned model'\n",
    "print('\\n------------ Result of {} ----------\\n'.format(name))\n",
    "print(classification_report(y_true, y_pred, digits=4))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.9733\n",
      "f1-macro score: 0.9100\n"
     ]
    }
   ],
   "source": [
    "print(\"Accuracy: {0:.4f}\".format(accuracy_score(y_true, y_pred)))\n",
    "print('f1-macro score: {0:.4f}'.format(f1_score(y_true, y_pred, average='macro')))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Confusion matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAfgAAAHCCAYAAAADydu0AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzs3Xd8lfX5//HXRWIQCArIzAkBwgoJIJAEKEUEF8oQRGYdRFC07lFr/bVVaK0iaBHEUfyquEGtFgHZ7gEBBC2gMgRLEvYSEEw4XL8/zp1wiNln5uR6Ph7nkXOv933dJ+OTe35EVTHGGGNMZKkW6gKMMcYY43/WwBtjjDERyBp4Y4wxJgJZA2+MMcZEIGvgjTHGmAhkDbwxxhgTgayBN6YcRKSGiMwVkUMi8paIXCUii0NdVyCIiIpIqxCsd7yIvFrGeT8SkesDXZMxlZE18KbSEpFtInJMRI6IyAERmS8iTX3I6y0iWaXMNhRoBJyjqsNU9TVVvaSi6yylnpki8lAgso0xkc8aeFPZDVTVWKAJsAt4siIhIhJdxlmbARtV9URF1mOMMcFiDbyJCKp6HHgbSM4fJyLVReQxEfmfiOwSkWdFpIYzrbeIZInIfSKyE3gDWADEOUcEjohInPc6RGQC8AAwwpk+VkQyROQzr3lURG4SkU0iclBEnhIR8Zo+RkS+dY44LBKRZkVtj4iMA64C/uisa65Xfiuv+Qr28r226R4R2S0iO0TkurJ8Hs70e51lckRkTEmft3No/CER+SK/PhE5R0ReE5GfRGSliDT3mr+HM+6Q87WH17QWIvKxiBwWkSVA/ULr6u6s56CIfC0ivUuqzRjjYQ28iQgiUhMYASz3Gj0RaAN0AloBLjwNdL7GQD08e+XXApcBOaoa67xyvNehqg8CDwOznenPF1POACAd6AgMB/o6NQ4C/h8wBGgAfIrnH4tfUdUZwGvAJGddA8vyOTjbdLazrWOBp0SkrjOt2M9DRC4F/gBcDLQGLirDukYC1zg5LYEvgRfxfKbfAg862fWA+cA04Bzgn8B8ETnHyXkdWI2nYf87MDp/BSLicpZ9yMn9A/BvEWlQxs/DmCrLGnhT2f1HRA4Ch/A0TpMBnL3mccBdqrpfVQ/jaZxHei17EnhQVX9R1WN+rGmiqh5U1f8BH+JpUAFuAh5R1W+dQ/wPA52K24uvoDzgb6qap6rvA0eAtmX4PIYDL6rqOlU9Cowvw7peVNUtqnoIz9GPLaq61Nm2t4DOznz9gU2q+oqqnlDVN4DvgIEikoDnn6G/Ot+HT4C5Xuu4GnhfVd9X1ZOqugRYBfSr4OdjTJVhDbyp7Aarah3gTOBW4GMRaYxnD7kmsNo5tHsQWOiMz7fHObTvbzu93v8MxDrvmwFTverZDwjgEpH/53Vq4Fkf1r2v0PUB+esv7fOIA7Z7LfdjGda1y+v9sSKG87c7roi8H/Hs+ccBB5x/KopadzNgWH7NTt098VxzYYwpQVkvLDImrKmqG3hHRP6FpwF4B08jk6Kq2cUtVsqwv20H/qGqrxUx7Qs8e9Sl1fMznoY6X2OgtCv/AfZS8uexA/C+AyGhDJlllYOnofaWgOcfjB1AXRGp5dXIJ3Bq27cDr6jqDX6sx5gqwfbgTUQQj0FAXeBbVT0JPAdMEZGGzjwuEelbQswu4BwROTtAZT4L3C8iKU49Z4vIsFLqSSw0bi3wOxGJcs6bn1+WFZfh83gTyBCRZOd6hgfLvFWlex9oIyK/E5FoERmB52LIear6I55D7hNEJEZEegLe1xu8iudQfl9nm890LiaM92N9xkQka+BNZTdXRI4APwH/AEar6npn2n3AZmC5iPwELAXaFhekqt/huejtB+dwcFxx81aEqr4LPArMcupZh+fCvuI8DyQ7tfzHGXcHngbwIJ6r7P9T3MJFKPbzUNUFwBPAB848H5Qjt0Squg/PhYf3APuAPwIDVHWvM8vvgG54Tlk8CLzstex2IP/ixD149ujvxf52GVMqUQ30UUljjDHGBJv9F2yMMcZEIGvgjTHGmAhkDbwxxhgTgayBN8YYYyKQNfDGGGNMBKqyD7qpX7++NmvWPNRlGGOMAb76avVeVQ15HwNRZzVTPeH7k6v12J5FqnqpH0qqsCrbwDdr1pzPV6wKdRnGGGOAGmdIWR6PHHB64hjV2w73Oef42qfqlz5XYFXZBt4YY4z5NQGJjLPXkbEVAbJ40UI6prQlJakVkydNtEzLtEzLtMwQZgaFACK+v8KBqlbJV5cuqXosT4t9HTl+QlskJuqG77fooaO/aIcOHfWrr9eXuExpL8u0TMu0TMss+gWsCnW7oKpIzYZ6ZuqdPr/CYXtsD74YKzMzadmyFS0SE4mJiWHYiJHMmzvHMi3TMi3TMkOQacrPGvhi5ORkEx9/qvdMlyue7Ozieh21TMu0TMu0zEBmBlWEHKKvNA28iDwiIn1EZLCI3F9oWlsReUlEqonIl6Gq0RhjTGXnXGTn6ysMhEcVZdMNWI6n/+tPCk07zxnXAU8XnD6Li3ORlbW9YDg7OwuXy2WZlmmZlmmZIcg05Rf2DbyITBaRb4B04EvgeuAZEXlARM4TkbXAJOAPwHygr4j4fIN7Wno6mzdvYtvWreTm5vLW7Fn0H3C5ZVqmZVqmZYYgM6gi5BB92N8Hr6r3isibwLXA3cBHqvpbr1k6OYflewAvAI+p6vqiskRkHDAOoGlCQonrjY6OZsrU6Qzs3xe3283ojDEkp6T4tC2WaZmWaZmWGeaEsDnE7itR1VDXUCoRuRFwAyuAO1T1eq9pNYH3VbW3iHwG9FbVE6VlpqamqT3JzhhjwkONM2S1qqaFuo5qsU20eofRPuccX/5oyLcnrPfgRaQTMBOIB/YCNT2jZS3wG2A2kATUcQ7jNwdWicgjqjo7JEUbY4wxYSCsG3hVXYvnEPwXQE88h+AnqeoGZ5bLReRe4AdgH9BPVf8YmmqNMcZEhAg5RB/2WyEiDYADqnoSSPJq3PP1Aj7DcyX9x8GuzxhjTIQJwkV2zu3da71eP4nInSJST0SWiMgm52tdZ34RkWkisllEvhGRLqWtI+wbeFXdo6r9nffdi5g+UFV3qerfVXV+8Cs0xhhjykdVv1fVTqraCUgFfgbeBf4ELFPV1sAyZxjgMqC18xoHPFPaOsK+gTfGGGOCJyQPurkQ2KKqPwKDgJec8S8Bg533g4CX1WM5nmvPmpQUGtbn4I0xxpigyu9NLrhGAm847xup6g7n/U6gkfPeBWz3WibLGbeDYlgDb4wxxnjzz0V29Qs9dG2Gqs741apEYoDLgfsLT1NVFZEK38tuDbwxxhjjf3vLeB/8ZcBXqrrLGd4lIk1UdYdzCH63Mz4baOq1XLwzrljWwPvR8i37/J7ZveU5fs8MxMONJEwezRgpTp4MxPfI75H2fa8E3AH4WYqqFsnfdwn2bXKjOHV4HuA9YDQw0fk6x2v8rSIyC0/fLIe8DuUXyRp4Y4wxxluQ/oERkVrAxcCNXqMnAm+KyFjgR2C4M/59oB+wGc8V99eVlm8NvDHGGBMCqnoUOKfQuH14rqovPK8Ct5Qn3xp4Y4wxJl8EdTYTGVsRIIsXLaRjSltSkloxedLEMi+3e0c2d107iIz+PcgY8FvefvlfAPx08AB/GHMlV/dN5w9jruTwoYOA55z4tIfu56pL0hl7eS82rv86KHUW5/jx45zXoxvdUjuRem57/j7hQZ8zwf91VvXMgwcPctXIYXTu0I4uHZNZsfxLn/Kq8vd9+/bt9L2oD507JtPl3BSmT5vqhyrDd9t/P24MzeMbkd65Q8G4/fv3M/CySzg3uQ0DL7uEAwcOhLzOkImQ7mKtgS+G2+3mzttvYc7cBaz5ZgNvzXqDbzcUfkpu0aKiovj9fX9j5vwveHrWQua89jzbNn/P689NpUv3Xry6aCVduvfi9ec8f0RWfLKU7B9/4NVFmdzzt38yZcK9QamzONWrV2fB4mWsWL2W5avWsGTxIjJXLPcpMxB1VuVMgHvvuZOLL+nLmv9+y/JVa2mb1M6nvKr8fY+OjmbipMdZ880GPv5sOf969qmwrNNfmVddk8F/5i44bdw/J0+k9wUX8PWGjfS+4AL+ObnijXKgfuaDIyQPugmI8KgiDK3MzKRly1a0SEwkJiaGYSNGMm/unNIXBM5p2Jg2KecCUDO2Ngkt27B31w6+WLaAvoNHANB38Ag+X/o+AJ8vW8Alg4YjIiR3SuPoT4fYt3tnwOssjogQGxsLQF5eHnl5eT7/RxqIOqty5qFDh/j8008Yfd1YAGJiYqhTp45PmVX5+96kSRM6d/E82rt27dokJbUjJ6fEO5BCUqe/Mnue14u6deudNm7+3Pe46mpPN6lXXT2aee9VvNZAbLspP2vgi5GTk018/KlbDl2ueLKzy/8LvzPrf2z+9r+0OzeV/fv2cE7DxgDUa9CI/fv2ALB31w4aNnEVLFO/cRx7d5V494Pf6yzM7XbTLa0zzVyNuPDCi+jatZtPeYGosypnbtu2lfoNGnDjDWP4Tdcu3HzT9Rw9etSnTKi633dvP27bxtq1a0ivYtu+e/cuGjfxPPm0UePG7N69q5Qlihfo71HA2SH64BCRwSKiIpLkDDcXkWMiskZEvhWRTBHJ8Jo/Q0Smh6xgL8eOHuGB2zO45f5/UCu29mnTRCSs7yGOiopixao1bNq6nVWrVrJ+3bpQl2S8uE+cYO2ar7hh3E18mfkVNWvW4nEfDqnmq+rf9yNHjjBq+JVMfvwJzjrrrFCXEzLh/vcp4OwQfdCMwtMd7CivcVtUtbOqtsPzDN87RaTUewLLIy7ORVbWqcf+Zmdn4XK5SljidCfy8njg9uu4aOBQel0yAIB65zQoOPS+b/dO6tarD0D9Rk3YvePUf7d7d+ZQv1GJfQj4rc7S1KlTh17n92bJ4oU+5QSiziqd6YrHFR9fsJd5xZChrF2zxqdMb1Xt+w6e0xKjhl/JiFFXMfiKIT7nVaZtB2jYsBE7d3iOHO7csYMGDRpWOCvQf5dM2YR1Ay8isUBPYCyehvxXVPUH4G7gdn+uOy09nc2bN7Ft61Zyc3N5a/Ys+g+4vEzLqiqT/nIHzVq2Yfh1NxeM73HBpSz6z2wAFv1nNj0uvKxg/OI5b6KqbFi7ilq1zyo4lB/IOouzZ88eDh70XOF/7NgxPli2lDZtk3zKDESdVTmzcePGxMc3ZeP33wPw0YfLSGrn20V2Vfn7rqrcdMNY2ia144677vYpK5B1BiIzX78BA3ntVU8nZq+9+hL9B1Y8N5B1Bpw/Ds+HydGPcL8PfhCwUFU3isg+EUkFinoe7FdAqX+JRGQcnn50aZqQUOK80dHRTJk6nYH9++J2uxmdMYbklJQyFb3uqxUsmfMmiW2SuX5wbwCuv+vPjLrhDibcNZb3//0qjeKa8uCU5wHofv7FrPhkKVdfkk71M2tw38PTyrQeX+sszs4dO7hhbAYn3W5OnjzJkKHD6Nd/gE+ZgaizKmcCPDZlGmMyriY3N5cWLRJ59rkXfMqryt/3Lz7/nNdfe4X27TvQLbUTABMeephLL+sXVnX6KzPjmt/x6ScfsW/vXtokNuXPfx3P3ff+iWt/N4KXX3yBpgnNePn12SGvM2TC5BC7ryQQzyX3FxGZB0xV1SUicjuQAEwH5qlqe6/56gI5qlrDOR+fpqq3lpSdmpqmn69YVdIs5WbPojf+Ys+iN/5SWZ5FX+MMWV3GzlkCqtrZTbV6D9+P4hxfeHfItyds9+BFpB5wAdDB6S4vClDgqSJm7wx8G8TyjDHGmLAWtg08MBR4RVULHsIvIh9zend5iEhz4DHgyWAWZ4wxJhIFvTe5gAnnBn4U8Gihcf8G7gdaisga4EzgMDBNVWc680QDvwSrSGOMMREmQk49hW0Dr6p9ihg3DSjtCrQUYFNAijLGGBPZIqizmbBt4CtCRBYAMcD4EJdijDHGhFRENfCqelmoazDGGFOZ2Tl4Y4wxJjJFyDn4yPg3xRhjjDGnsT14PwrEQ2kunPKJ3zOX3dXL75nGv6oF4EEigWAPTQp/gXgoTcSzQ/TGGGNMBIqQfzKtgTfGGGPySeRcZBcZW2GMMcaY01gDX4LFixbSMaUtKUmtmDxpYlhkVhN48douTBri6ZkpNaEOL1zbmZmju/D0qHNx1TkTgBFpLl69LpWXMrowdXgHGp1Vvczr2L59O30v6kPnjsl0OTeF6dOmlrvOooTj52mZgc08fvw45/XoRrfUTqSe256/T3jQD1VWjm0PROaN148hIa4hqZ3alz5zOVSGbQ+qCOku1hr4Yrjdbu68/RbmzF3Amm828NasN/h2w4aQZw5LdbFt388Fw3+4uBUT5n1HxktfseTb3WT8xtMN7qZdRxj7yhpGz/yKDzfu5ZbzW5R5HdHR0Uyc9DhrvtnAx58t51/PPhUW226ZlS+zevXqLFi8jBWr17J81RqWLF5E5orlYVdnZcm8ZnQGc+Yt9CmjsMqy7cEkIj6/woE18MVYmZlJy5ataJGYSExMDMNGjGTe3DkhzWwQG0OPxHrM/e/O08bXqu65lCK2ejR7j+QC8NX2Q/xy4iQA63N+okHtsu/BN2nShM5dugBQu3ZtkpLakZOTXeblixKOn6dlBj5TRIiNjQUgLy+PvLw8n/duKsu2ByKz53m9qFevnk8ZhVWWbTflZw18MXJysomPP9VxncsVT3a2b42cr5l3XNCSpz/eetqtSRMXbuSxK9vz7k3d6JvckFdWbP/VcgM7NGb5DwcqVPOP27axdu0a0rt2q9Dy+cLx87TMwGeCZ2+uW1pnmrkaceGFF9HVfpbCSlXe9qIItgcfUCIyWERURJKc4eYickxE1ojItyKSKSIZXvNniMgeEVkrIhtE5IaQFR8gPRLrceDnPL7fdeS08SPS4vnDv9dxxbMreH/dLm7vk3ja9EuSG5LUuDavr/x1w1+aI0eOMGr4lUx+/AnOOussn+o3VVdUVBQrVq1h09btrFq1kvXr1oW6JGOKJ356hYFwvU1uFPCZ8zX/qpwtqtoZQEQSgXdERFT1RWf6bFW9VUQaAutF5D1V3VXRAuLiXGRlnWoUs7OzcLlcFY3zObOj6yx6tjqH3yTWIya6GrViopg8JIVm59Rkw47DACz7bg+PDz118U1aszqM7p7ALbO+Js9dvgeS5OXlMWr4lYwYdRWDrxhSrmWLEm6fp2UGJ9NbnTp16HV+b5YsXkhK+4pfJFZZtj3Qn6e/VOVtL1r47IH7Kuz24EUkFugJjAVGFjWPqv4A3A3cXsS03cAWoJkvdaSlp7N58ya2bd1Kbm4ub82eRf8Bl/sS6VPms59u44pnVzB0RiYPzv2W1f87yJ/eXU+tmGia1q0BQHrzuvy433MBXuuGtfjjJa257511HPw5r1x1qio33TCWtkntuOOuu8u3kcUIt8/TMoOTuWfPHg4ePAjAsWPH+GDZUtq0TQq7OitLZiBU5W2PdOG4Bz8IWKiqG0Vkn4ikAvuKmO8r4Fd/KZy9+0RgcxHTxgHjAJomJJRYRHR0NFOmTmdg/7643W5GZ4whOSWl/FsTwEy3wqOLN/KPQcmcVOXw8RM8snAjALf0TqTGGVE8NCgZgF0//cJ9764vU+4Xn3/O66+9Qvv2HeiW2gmACQ89zKWX9atwrZXh87RM/2fu3LGDG8ZmcNLt5uTJkwwZOox+/QeEXZ2VJfPaq0fx6ccfsXfvXlo2j+evD0wgY8zYsKszEJnBFCl78BKIZ0n7QkTmAVNVdYmI3A4kANOBeara3mu+ukCOqtZwzsdPBrKBX4CJqvpuSetJTU3Tz1esCtRm+I09i96EM3sWvfGXGmfIalVNC3UdUfVaaK1LJvicc3j26JBvT1jtwYtIPeACoIOIKBAFKPBUEbN3Br71Gp6tqrcGvkpjjDEm/IVVAw8MBV5R1RvzR4jIx0BT75lEpDnwGPBkMIszxhgT+SLlKFK4NfCjgEcLjfs3cD/QUkTWAGcCh4FpqjozuOUZY4yJaGF0m5uvwqqBV9U+RYybBkwrZbmZwMzAVGWMMaaqELtNzhhjjDHhLKz24I0xxphQi5Q9eGvgjTHGGC+R0sDbIXpjjDEmAtkevB8F4qEfS+88z++Z2/Yc9Xtm8wa1/J5ZlVWWB8hEyp6OMd6C9XMtInWA/wPa43nmyxjge2A20BzYBgxX1QPiKWoq0A/4GchQ1a9Kyrc9eGOMMSZfcHuTm4rn0exJwLl4Ht72J2CZqrYGljnDAJcBrZ3XOOCZ0sKtgTfGGGO8BKM/eBE5G+gFPA+gqrmqehBPfywvObO9BAx23g8CXlaP5UAdEWlS0jqsgTfGGGOCrwWwB3hRRNaIyP+JSC2gkarucObZCTRy3ruA7V7LZznjimUNfAkWL1pIx5S2pCS1YvKkiT7nHT9+nPN6dKNbaidSz23P3yc8WPpCZZDUugXpnTvSLa0zv+2eXubl/nzX7/lth+YM7HNqmcl/+zP9zuvMoAu7ceuYkfx06GDBtBlPPkbfHh25rGdnPvtoabnr9PfnWZUzA/WzVBm23TKrbmYw5D/oxg978PVFZJXXa1yhVUUDXYBnVLUzcJRTh+MBUM/FOBW+IMca+GK43W7uvP0W5sxdwJpvNvDWrDf4dsMGnzKrV6/OgsXLWLF6LctXrWHJ4kVkrljul3oXLPmAFavW8PnylWVeZvCIq5jx2n9OG9ej1wW89+FK5ixbQfPE1sx48nEANm/8lvfnvM3cD1fy3Ovv8rf778Ltdpd5XYH4PKtyZiB+lirLtltm1cwMJj818HtVNc3rNaPQarKALFVd4Qy/jafB35V/6N35utuZns3p/bLEO+OKZQ18MVZmZtKyZStaJCYSExPDsBEjmTd3jk+ZIkJsbCwAeXl55OXlQQivQk7v3pM6deueNu63vS8kOtpzc8W5qens2uH5+flg0Xz6DRpKTPXqxCc0J6F5It+sKXt3u4H4PKtyZiB+lirLtltm1cwMqiBcZKeqO4HtItLWGXUhsAF4DxjtjBsN5H9w7wHXikd34JDXofwiWQNfjJycbOLjT/2z5HLFk51d4j9LZeJ2u+mW1plmrkZceOFFdO3azedMEWFgv7706JbG8/9X+J/EinvnjVc474JLANi1I4fGcfEF0xo1cbF7Z06ZswLxeVblTPD/z1Jl2XbLrJqZEeo24DUR+QboBDwMTAQuFpFNwEXOMMD7wA/AZuA54ObSwsPuPngRGQy8C7RT1e+crmG/Bb7jVE9yT6vqTGfaZ0CCqp70ylgL3Oh16CNsREVFsWLVGg4ePMjIYUNYv24dKe3b+5S59MNPcblc7N69m4GXXULbtkn0PK+XT5nPTp1EVHQUA4eM8CnHBE4gfpaMqfIkePfBq+paIK2ISRcWMa8Ct5QnPxz34EfhabRHeY3boqqdVbUdMBK4U0SuU9VtwP+AgqfBiEgSUNvXxj0uzkVW1qkLFrOzs3C5SrxgsVzq1KlDr/N7s2TxQp+z8utq2LAhAwcNZtXKTJ/y3p39Kh8tXcjk6S8U/KA3ahLHzpysgnl27cimYeO4MmcG4vOsypne/PWzVFm23TKrZmYwBeM2uWAIqwZeRGKBnsBYPA35r6jqD8DdwO3OqDcKzTsSmOVrLWnp6WzevIltW7eSm5vLW7Nn0X/A5T5l7tmzh4MHPVelHzt2jA+WLaVN2ySfMo8ePcrhw4cL3i9buoTklIrvxX364RKef3oKT8+cTY2aNQvG97mkH+/PeZvcX34h63/b+HHrFjp2Luofz6IF4vOsypmB+FmqLNtumVUz05RfuB2iH4TnqT4bRWSfiKQC+4qY7ysg/6/Zm8BaEblNVU8AI4BhRYU7tymMA2iakFBiIdHR0UyZOp2B/fvidrsZnTGG5JSUim2VY+eOHdwwNoOTbjcnT55kyNBh9Os/wKfM3bt2MXLYEABOnDjB8JGjuKTvpWVa9p7fZ5D55acc3L+P3qltuPWeP/Pc9MfJ/eUXxo7w/DKem5rO+Een0bptMpcOHMKA3mlERUXz14f/SVRUVJnrDMTnWZUzA/GzVFm23TKrZmYwhcseuK8kEM+8rigRmQdMVdUlInI7kABMB+apanuv+eoCOapawxleAjwG7AJeUdUOpa0rNTVNP19R9qvAyyKcPsuS/Lj3Z79n2rPo/auyPIveGH+pcYasVtWyHxYMkJgGrbT+kEk+5+yYcWXItyds9uBFpB5wAdBBRBSIwnOD/1NFzN4Zz4V3+fIP0+9y3htjjDEVEyH/C4dNAw8MxbP3fWP+CBH5mNNv7Me5cv4x4Emv0e8Aj+DpYedXVx8aY4wxVU04NfCjgEcLjfs3cD/QUkTWcOo2uWmqOjN/JlU9KCJfAo2di/CMMcaY8gvibXKBFjYNvKr2KWLcNGBaGZcfXPpcxhhjTMkipYEPq9vkjDHGGOMfYbMHb4wxxoSDSNmDtwbeGGOM8RYZ7bs18MYYY4w324M3v1JZfigC8VCaRRt2+j2zb3Jjv2dWFpXlZ8kYE76sgTfGGGMc4dRZjK+sgTfGGGO8REoDb7fJlWDxooV0TGlLSlIrJk+aaJllsGdnNn8ZeyW3XtGL2644n7mvPQfAa9Mf5Y6hF3Dn8It48MYR7N99+iH9TevWMqRLPF8smReUOi3TMi2z8mWa8gmrzmaCqbTOZtxuNx2S2zB/wRJc8fH07J7OS6++Qbvk5AqvM5Iz88/B79+ziwN7d9GyXUeOHT3CPSP7cv8TL3BOozhqxtYGYN5r/8f2Hzby+79OKljf+BtHcEb16lw0eBQ9Lvb0ilbaOfhw2XbLtEzL9D0zXDqbqd6otcaNesLnnG1TB4R8e2wPvhgrMzNp2bIVLRITiYmJYdiIkcybO8cyS1GvQSNatusIQI1ascQntmbf7p0FjTvA8eM/n3YIbP4bz/Obi/pzdr36QavTMi3TMitXZlCJH15hwBr4YuTkZBMff6qfG5crnuzsbMssh13Z2/nhu//SpkMXAF598hHGXpLKJ/PfYdTN9wKwb9cOVnywgEuHjw5ZnZZpmZYZ/pnBlH+hnS+vcBB2DbyIDBYRFZEkZ7i300+89zwzRWQ2+conAAAgAElEQVSoiESJyGoR6eU1bbGIDAt23eZ0x34+yqP3jGXsvX8r2Hu/+rb7eX7xanr1H8L7s14E4PnJD3DtnX+hWrWw+1E0xphKLRz/qo4CPnO+lkhV3cDNwHQROUNERgEnVfUtX4uIi3ORlbW9YDg7OwuXy2WZZXAiL49H7x7L+f2G8JuL+v9q+vn9hvDl0vkAbF7/NY/ddxM3XJbOl0vm8a9//InlHywISp2WaZmWWXkyg0ZsDz4gRCQW6AmMBUaWZRlVXQF8CYwHHgZu9UctaenpbN68iW1bt5Kbm8tbs2fRf8DlllkKVWX6+LuJT2zNoGtvKhif8+OpXnxXfLgIV4tWAMxYkMlzC1by3IKV/ObiAdz454l0v+CygNdpmZZpmZUrM1gEEPH9FQ7C7T74QcBCVd0oIvtEJLWMy90PbAeeUNXN/igkOjqaKVOnM7B/X9xuN6MzxpCckmKZpfh2TSYfzXubZq3bcefwiwDPofml775OzrYtSLVqNGgSz+//8qhPNfpap2VapmVWrkxTfmF1m5xzrn2qqi4RkduBBGAucK+qDvCabyYwV1X/7QwPBp4GVqrqoBLyxwHjAJomJKRu3PJjwLalqrFH1RpjfBEut8md2biNNr1mms85mx+7LOTbEzZ78CJSD7gA6CAiCkQBCrwE1C00ez1gr7NcLWCSs+yLItJPVd8vah2qOgOYAZ774AOxHcYYYyq3cDnE7qtwOgc/FHhFVZupanNVbQpsxdOYx4lIOwARaQacC6x1lnsAeFNVv8Nzwd0UETkz+OUbY4yJBJFykV3Y7MHjuWq+8InZf+O52O5qPHvnZwJ5wPWqekhEUoAr8DT4qOoaEVkE3AdMCFrlxhhjTJgJmwZeVfsUMc77REj3IqavB9oUGne7/6szxhhTJYTRVfC+CpsG3hhjjAk1AapVi4wW3hp4Y4wxxkuk7MGH00V2xhhjjPET24M3xhhjvITLVfC+sgbeGGOMyWcX2RlzukA8dS7nwDG/Z8bVreH3TGOMCUfWwBtjjDEOT2czkbELbw28McYYUyB8nkTnK7uKvgSLFy2kY0pbUpJaMXnSxCqVeeP1Y0iIa0hqp/Z+yQP/1fnijKe4tFcal56Xyov/mg7A1EkP0aNjSwb06caAPt34cOnCkNdpmZZpmf7NDJZI6S42rHqTC6bU1DT9fMWqYqe73W46JLdh/oIluOLj6dk9nZdefYN2yckVXmdlyQT47NNPqFUrluvHXMvqtet8yqponUWdg//+2/XcceNo3l34CWfExHDdiEH8ffI05rz9BjVrxXLDLXeWWEdp5+Ary/fIMi0z0jLDpTe5mnFttc24p33O+XrCRSHfHtuDL8bKzExatmxFi8REYmJiGDZiJPPmzqkSmQA9z+tFvXr1fM7J5686t2z6nk5d0qhRsybR0dF07dGTRfN9315/12mZlmmZ/v+7FCyR0tmMNfDFyMnJJj6+acGwyxVPdnZ2lcgMBH/V2SYpmZXLv+DA/n0c+/lnPl66iB05WQC88sKz9Du/K/fdcSOHDh4IaZ2WaZmWGf5/l4rkh8PzYdK+B6eBF5HGIjJLRLaIyGoReV9E2jiv90Vkk4h8JSJvikgjZ5meIpIpIt85r3FeeeNF5GcRaeg17khR701kadUmiRtvu5vRwwdy3chBtGvfkaioKK7KuIEPM9cz78PlNGjUmIcf/FOoSzXGmBKJyDYR+a+IrBWRVc64eiKyxGkXl4hIXWe8iMg0EdksIt+ISJfS8gPewIvnWMW7wEeq2lJVU4H7gUbAfOAZVW2tql2Ap4EGItIYeB24SVWTgJ7AjSLS3yt6L3BPoOqOi3ORlbW9YDg7OwuXy1UlMgPBn3UOvyqD95Z+waz3lnB2nTq0SGxF/YaNiIqKolq1aoy8egxfr1kd8jot0zIt03+ZwZJ/m1wQD9H3UdVOXufr/wQsU9XWwDJnGOAyoLXzGgc8U1pwMPbg+wB5qvps/ghV/RpPkV+q6lyv8R+p6jrgFmCmqn7ljN8L/JFTGwrwAjBCRPx3othLWno6mzdvYtvWreTm5vLW7Fn0H3B5lcgMBH/WuXfPbgBysrazaP57XH7lCHbv2lEwffH779EmqWIXCFWW75FlWmZVywymEB+iHwS85Lx/CRjsNf5l9VgO1BGRJiUFBeM++PZAUbtTxY0HSOHUBuZb5YzPdwRPI38H8KCPNf5KdHQ0U6ZOZ2D/vrjdbkZnjCE5JaX0BSMgE+Daq0fx6ccfsXfvXlo2j+evD0wgY8zYsKjzljG/4+CB/URHn8H4iVM46+w6TLj/Hjas/wZBiE9I4KHHngx5nZZpmZbp379LweKni+Tq5x92d8xQ1RmF5lFgsYgo8C9neiNVzd9j2YnnaDeAC9jutWyWM24HxQj4bXIicjvQQlXvKjT+n8CPqjq1iGXeAV5S1Tle484GtqpqPREZj6eB/z9gLdAB2KGqsc68R/LfF8odh+fQBk0TElI3bvnRT1tpAsEeVWtM1REut8nVcrXV5Jv/5XPOqr/0KXV7RMSlqtnO9WRLgNuA91S1jtc8B1S1rojMAyaq6mfO+GXAfapa7P3ewThEvx5ILcd4gA1FTEt1limgqgfxnKu/pSyFqOoMVU1T1bQG9RuUZRFjjDFVTLAO0atqtvN1N55r1boCu/IPvTtfdzuzZwNNvRaPd8YVKxgN/AdA9UJXwXcENgI9vC+cE5FeItIeeArIEJFOzvhzgEeBSUXk/xO4EXvsrjHGGF9JcC6yE5FaIlI7/z1wCbAOeA8Y7cw2Gsg/kv0ecK1zNX134JDXofwiBbyBV885gCuAi5zb5NYDj+A5tzAAuM25HWADcDOwxyn6auA5EfkO+AJ4wfuCPK/8vXj+86kOICLRwC+B3i5jjDHGB42Az0TkayATmK+qC4GJwMUisgm4yBkGeB/4AdgMPIenvSxRUPZ6VTUHGF7M5EuLWeYTIL2YaeMLDd8N3O0MpgBbKlSoMcaYKs1zm1zg16OqPwDnFjF+H3BhEeOVMp6OzhdRh7VF5CbgdqDkB5IbY4wxRQqfR836KqIaeOde+2dLndEYY4wpRoS07/YsemOMMSYSRdQevDHGGOMrO0RvjDHGRJow6g3OV9bAm7AViKfODX0+0++Zb4/t6vdMY4zxlTXwxhhjjCO/N7lIYA28McYY4yVSGni7ir4EixctpGNKW1KSWjF50sTSF7DMSpdZTWDqlSk8cGkbAG4/vwVPDm3Pk0Pbc//FrTgz2vMr0iA2hn8MaMuTQ9vzyMAkzql1RlDrtEzLtMzgCXF3sX5jDXwx3G43d95+C3PmLmDNNxt4a9YbfLthg2VGWObl7Ruz/cDxguHnvviR295ex21vr2PPkVwGtPf01Di2ewLLNu7jtrfX8cbqbEZ3bVpcZEDqtEzLrOqZpvysgS/GysxMWrZsRYvERGJiYhg2YiTz5s4pfUHLrDSZ59Q6g/RmZ7P4u90F447lnSx4HxMl5Pem3LTumXyT/RMA3+QcpnvzukGr0zIt0zKDKxidzQSDNfDFyMnJJj7+1F6ayxVPdnaJPfNZZiXLHNejGS8s317QiOe7o3cLXrmmM/F1ajBv/S4Atu47Ro8Wnkb9Ny3qUjMmitrVy3YJSzhuu2VaZmXLDBo/HJ4Pk/Y9/C6yExE38F/gDOAE8DIwRVVPikhvPF3nbfVa5BHgfud9Y8AN7HGGu6pqbjDqNpVLekIdDh7LY8ven+nQpPZp06Z+tJVqAjf+thnntazH0u/38sLy/3FTz2Zc2LY+63ccZu+RXE4W/s/AGFPpiT2LPqCOqWp+P/ANgdeBs4AHnemfquqAQsvMduYfDxxR1cd8LSIuzkVW1vaC4ezsLFwul2VGSGZy41i6NatLWkIdYqKEGmdEcc8FiTz+wQ8AnFT4ZMt+rjy3CUu/38v+n/N4ePFmAM6MrkaPFvU4musOeJ2WaZmWaSoqrA/Rq+puYBxwqwT5X6q09HQ2b97Etq1byc3N5a3Zs+g/4HLLjJDMlzKzyHhtLWNf/5pJS7fwTc5hHv/gB5qcVb1gnm7N6pB18BgAZ50ZTf4P4LDOcSz5fk8Rqf6v0zIt0zKDzw7RB4mq/iAiUUBDZ9R5IrLWa5YrVdXv/b9HR0czZep0Bvbvi9vtZnTGGJJTUiwzgjMFuKtPIjXPiEIEtu77mac+3QZAhya1Gd2tKaqwbsdPPPPZjyGr0zItsypmBlO1cGmhfSQaZucRReSIqsYWGncQaAu0A/5QxCH6/PnGU8IhehEZh+eIAE0TElI3bin7H2kTGexRtcaEpxpnyGpVTQt1HWcltNNuf3zR55ylt/0m5NsT1ofoAUQkEc+Fc7tLm7c0qjpDVdNUNa1B/Qa+F2eMMSbi2CH6IBCRBsCzwHRV1Ui5stEYY0x48jTQkdHWhGMDX8M5x55/m9wrwD+9phc+B/+Qqr4dzAKNMcZErmqR0b6HXwOvqlElTPsIOLuE6eMDUJIxxhhT6YRdA2+MMcaEkh2iN8YYYyJQhLTv4X8VvTHGGGPKz/bgjTHGGIfgeR59JLAG3hhjjPFiV9EbUwnZU+eMMSUKo/7cfWXn4I0xxpgIZHvwxhhjjJcI2YG3PfiSLF60kI4pbUlJasXkSRMt0zIrZPv27fS9qA+dOybT5dwUpk+b6ocqK8e2W6Zl+jMzGARPb3K+vsJB2PUmFyypqWn6+YpVxU53u910SG7D/AVLcMXH07N7Oi+9+gbtkpMrvE7LrJqZO3bsYOeOHXTu0oXDhw/To1sqb779n7Cr0zItM5SZ4dKbXN3mydrnr6/4nPPu9Wkh3x7bgy/GysxMWrZsRYvERGJiYhg2YiTz5s6xTMsstyZNmtC5SxcAateuTVJSO3JyssOuTsu0zHDODKZI6U3OGvhi5ORkEx/ftGDY5YonO9u3P8qWWTUzvf24bRtr164hvWs3n3Iqy7ZbpmX6KzOYxLmS3pdXOAjLBl5Ejjhfm4uIishtXtOmi0iGiDwlImtFZIOIHHPerxWRoSIyU0SGhm4LjPm1I0eOMGr4lUx+/AnOOuusUJdjjCmCP/bew6R9L/4qehEp8S+Qqv7k/3KKtBu4Q0T+paq5Xuu/BTz/BADzVLVT/jQRGeDrSuPiXGRlbS8Yzs7OwuVyWaZlVkheXh6jhl/JiFFXMfiKIT7nVZZtt0zL9FemKb+S9uDXA+ucr+sLDa8LfGkF9gDLgNFBXCdp6els3ryJbVu3kpuby1uzZ9F/wOWWaZnlpqrcdMNY2ia144677vYpK5B1WqZlhnNmMEXKVfTF7sGratPipoXAo8ACEXkhWCuMjo5mytTpDOzfF7fbzeiMMSSnpFimZZbbF59/zuuvvUL79h3oluo50DThoYe59LJ+YVWnZVpmOGcGU3g0z74r021yIjISSFTVh0UkHmikqqsDVpTIEVWN9Tr83l5EXgaWAN2AVao605m3YB6v5Wc6494ulDsOGAfQNCEhdeOWHwO1CcYYY8ohXG6Tq9ciWS8Z/5rPObMzuoR8e0q9yE5EpgN9gGucUT8DzwayqGI8DNyHD/9cqeoMVU1T1bQG9Rv4rzJjjDERoypdRd9DVW8EjgOo6n4gJqBVFUFVvwM2AAODvW5jjDFVg+dJdr6/yrw+kSgRWSMi85zhFiKyQkQ2i8hsEYlxxld3hjc705uXll2WBj5PRKoB6qzkHOBk2cv3q38A8WWYLxr4JcC1GGOMiTR+2Hsv5x78HcC3XsOPAlNUtRVwABjrjB8LHHDGT3HmK1FZGvingH8DDURkAvBZWYJ9oaqxztdt3ufWVfVrVa2Wf/69qHmcf0baAVsCWaMxxhjjC+eatv7A/znDAlwA5F8/9hIw2Hk/yBnGmX6hlPKfRKm9yanqyyKyGrjIGTVMVYN5m1yZiUgcsBT4UFU3hLoeY4wxlU8QT6E/AfwRqO0MnwMcVNUTznAWkP8AARewHUBVT4jIIWf+vcWFl7W72CggD89h+rB8+h2AquYAFe8hwRhjTJXnp4vk6ouId49mM1R1htc6BgC7VXW1iPT2xwoLK7WBF5E/A78D3sVz/cHrIvKaqj4SiIKMMcaYUMm/yM4P9pZym9xvgctFpB9wJnAWMBWoIyLRzl58PJD/EP9soCmQJSLRwNnAvpIKKMve+LVAuqr+RVX/DHQFMsqwnDHGGGOKoKr3q2q8qjYHRgIfqOpVwIdAfl8qo4H8bvje49QTXYc685f4IJuyHKLfUWi+aGecMcYYE3FCfB/7fcAsEXkIWAM874x/HnhFRDYD+/H8U1CikjqbmYLnnPt+YL2ILHKGLwFW+lR+GFA8zwgPd+HywAQTXHXTb/V75oGV0/2eWVkE4ne9svxuVuVtr6hgb52qfgR85Lz/Ac+R8sLzHAeGlSe3pD34/Cvl1wPzvcYvL88KjDHGGBN8JXU283xx04wxxphIJELY9Abnq7I8i76liMwSkW9EZGP+KxjFhdLx48c5r0c3uqV2IvXc9vx9woN+y3a73XRP78KQwf556u7iRQvpmNKWlKRWTJ400S+ZN14/hoS4hqR2al/6zCHMDMS2h1Nm62YNWT7rTwWvXZ9O5tbf9eaBm/uTOft+ls/6E3OfvoUmDc4G4K5rLyyYd9Vb/48jq6ZR96yaAa+zsmcG6ve9Mmw7+P9vEgSmzmAR8f0VDkrtTU5EPgUeAh7D80Sd6wBV1b8GvrzA6ZKapp8vL/5SAlXl6NGjxMbGkpeXx4W9z+Oxfz5B127dfV73tCf+yVerV/PT4Z945z9zS5y3tHNdbrebDsltmL9gCa74eHp2T+elV9+gXbJvjwP47NNPqFUrluvHXMvqtf55rpG/MwOx7eGSWdQ5+GrVhC2L/sH5107mwE/HOHz0OAA3jzqfpMQm3P6PWafN369Xe267qg+X3fgkUPo5+HDZ9kBklvZ3riK/76H43QzEtkP5/iZBYLY9XHqTa9AyRa+Y+KbPOc8Nbx/y7SnLbXI1VXURgKpuUdW/AJcFtqzQExFiY2MByMvLIy8vzy//lmVlZbFwwftkjBlb+sxlsDIzk5YtW9EiMZGYmBiGjRjJvLlzSl+wFD3P60W9evX8UGHgMgOx7eGc2adrW7Zm7eF/Ow4UNO4ANWtUL/KP+PBL03hzYdl7dQ7nbQ90ZiB+3yvLtvv7bxIE7u+SKZ+yNPC/OM933yIiN4nIQE49Vi+iud1uuqV1ppmrERdeeBFdu3bzOfOP99zFQ488SrVq/nkgYE5ONvHxTQuGXa54srOzS1gicgRi28M5c1jf1NMa7PG3DGTTgr8z8rI0/v7M/NPmrXHmGVzcox3/WbY26HVWxkzw/+97Zdl2f/9Ngsr/dylSDtGX5Tt6F1ALuB3Pk3duAMaUd0UioiLyuNfwH0RkvNfwOBH5znllikhPr2nbRKS+13Bvr671MkTkpIh09Jq+rixd6ZUmKiqKFavWsGnrdlatWsn6db4dVn5//jwaNGxAly6pvpZmqpgzoqPof34H3lmypmDc+Kfm0vqyvzJrwSpuGtHrtPn79+rAl2t/4MBPPwe71ErL37/vlYH9Tfo1Qagmvr/CQakNvKquUNXDqvo/Vb1GVS9X1c8rsK5fgCHeDXU+55m8NwI9VTUJuAnPI3EblzE7C/hzBWoqkzp16tDr/N4sWbzQp5zlX3zO/HlzSWrdgmuvHsXHH37AmNHX+JQZF+ciK2t7wXB2dhYul6uEJSJHILY9XDP79kxm7Xfb2b3/8K+mzX5/JYMv7HTauGF9U3mrHIfn/VVnZc305q/f98qw7YH4mxSIOk3FFNvAi8i7IvJOca8KrOsEMAPPEYHC7gPuVdW9AKr6FZ5u8W4pY/Y8IEVE2lagriLt2bOHgwcPAnDs2DE+WLaUNm2TfMr82z8eYfPW7Xy3aSsvv/oG5/e5gBdeesWnzLT0dDZv3sS2rVvJzc3lrdmz6D/gcp8yK4tAbHu4ZhY+n94yoUHB+wG9O7Jx266C4bNiz6RnaivmfvRN0OusrJmB+H2vDNseiL9JgagzqPxweD5MduBLfNBNIB579RTwjYhMKjQ+BSi8u7GKU8/dLc1JYBLw/8qxTIl27tjBDWMzOOl2c/LkSYYMHUa//gP8Ee1X0dHRTJk6nYH9++J2uxmdMYbklBSfc6+9ehSffvwRe/fupWXzeP76wASfL8Lxd2Ygtj0cM2ueGcMF3ZK49aE3CsY9dPsgWjdryMmTyv927D/tCvrL+5zLsuXf8fPx3KDWWZkzA/H7Xlm2PRAqS53FiZQn9ZV6m5zfViRyRFVjReRveLqePQbEqup4EdkPtFDVQ17zDwJGq+oQEdmKp8Obvc603sA9qjpQRDKANOBOPE/duxSYCwxQ1W2FahgHjANompCQ+v3m0yaHpUj5QTPlY4+q9a+q/LjWyrLt4XKbXMNW7XXE5Ld8zpk+JDnk2xOKvt2fAMbiuXAv3wag8FUeqXgabPB0iVfXa1o9CnVy73St9ziew/1FUtUZqpqmqmn16zcobjZjjDGm0gt6A6+q+4E38TTy+SYBj4rIOQAi0glPl7RPO9M/Aq5xpkUBV+PpUq+wmcBFgLXexhhjyk3wHKHw9RUOytzAi0h1P673caDganpVfQ94AfhCRL4DngOuVtX8bmn/DrQSka/xdJ+3GXi1cKiq5gLTgIZ+rNUYY0wVUk18f4WDUvuDF5GuePqhPRtIEJFzgetV9bbyrEhVY73e7wJqFpr+DPBMMcseAn5XzLSZePbc84en4WnkjTHGmCqrLHvw04ABeM6Do6pfA30CWZQxxhgTKlVmDx6opqo/Fjqn4A5QPcYYY0zIeO5jD5MW2kdlaeC3O4fp1bnA7TYg4ruLNcYYUzWFyx64r8pyiP73wN1AArAL6O6MM8YYY0yYKnUPXlV3AyODUIsxxhgTchFyhL5MV9E/B/zqUUiqOi4gFQVJ/r2OxoSjQDx17sDR8j26tizq1orxe2YgVOXf9aq87RUhEDa9wfmqLOfgl3q9PxO4AthezLzGGGOMCQNlOUQ/23tYRF4BPgtYRcYYY0wIheIZ7oFQke1oATTydyHhaPGihXRMaUtKUismT5pomZYZkZmHDh7khmtHcl56B3p17ciqzOXM/c+/6d29E666Z/L1mvL1Kx+oOi3TMoMlUrqLLbU3ORE5wKlz8NWA/cCfVPXNANcWUKmpafr5ilXFTne73XRIbsP8BUtwxcfTs3s6L736Bu2Skyu8Tsu0zFBmFncO/o6bxtK1x2+56tox5Obmcuznn9m9awdSrRr33XkrDzw0kXM7F+4LyqO0c/Dhsu2WGf6Z4dKbXFybDjp22js+5zx0WZuQb0+Je/DiuTrjXDydtzQA6qpqYmVv3MtiZWYmLVu2okViIjExMQwbMZJ5c+dYpmVGVOZPhw6x/ItP+d011wEQExPD2XXq0LptO1q1butTjf6s0zIt05RfiQ28enbv31dVt/MKTufxYSAnJ5v4+KYFwy5XPNnZ2ZZpmRGV+b8ft3FO/QbcdfMNXHxeV+657SZ+PnrUp9oCUadlWmYwRcoh+rKcg18rIp0DWYSIqIg87jX8BxEZ77yfKSJDC81/xPna3Fn2Ia9p9UUkT0T8f5+RMRHG7T7Bf79ew7Vjx7Hk00xq1qzJ9CmTQ12WMSEVKc+iL7aBF5H8K+w7AytF5HsR+UpE1ojIV36u4xdgiIjUL3XOX9sK9PcaHgas97WguDgXWVmn7gbMzs7C5XJZpmVGVGaTOBdN4uLpktYVgAGDhvDfb9b4VFsg6rRMywyW/PvgfX2Fg5L24DOdr5cDbYF+eBrPoc5XfzoBzADuqsCyPwPfikj+xQwjAJ+vEUhLT2fz5k1s27qV3Nxc3po9i/4DLrdMy4yozIaNGhMXH8/mTd8D8OnHH9K6bTufagtEnZZpmab8SroPXgBUdUuQankK+EZEJlVg2VnASBHZhaenuxwgzpdioqOjmTJ1OgP798XtdjM6YwzJKSm+RFqmZYZl5kOPTuHWGzLIy80loXkLpjz9HAvmzuEv993Fvr17uGb4YFI6dOSNd+aHtE7LrNqZwRQmO+A+K/Y2ORHJAv5Z3IKqWuy0chchckRVY0Xkb0AecAyIVdXxIvIiMF9V3/aa/7Cq1haR5sA8oAuwEngVOATkAmmqemuh9YwDxgE0TUhI3bjlR39tgjFhryo/qtaEv3C5Tc7VtoPe/PS7Puf85aLWId+ekg7RRwGxQO1iXoHwBDAWqOU1bh9QN39AROoBe70XUtVcYDVwD/A2xVDVGaqapqppDeo38GfdxhhjTFgp6RD9DlX9W9AqAVR1v4i8iaeRf8EZ/RFwp4i85DTkGcCHRSz+OPCxkxGMco0xxkQgITLakFLPwYfA40DBoXVVnSciqcBqEXEDW4CbCi+kquvxw9Xzxhhjqi7PVfShrsI/SmrgLwxWEaoa6/V+F1Cz0PQJwIQiltsGtC9i/Exgpp/LNMYYUwVESgNf7Dl4Vd0fzEKMMcYY4z9l6Q/eGGOMqTIi5Toua+CNMcYYRySdg4+Ufu2NMcaYSkNEzhSRTBH5WkTWi8gEZ3wLEVkhIptFZLaIxDjjqzvDm53pzUtbhzXwxhhjTD4/9CRXxiP8vwAXqOq5QCfgUhHpDjwKTFHVVsABPLeN43w94Iyf4sxXIjtEb0wVEYinzu0/4v+n49WLtafjmdAKRmcxTvfrR5zBM5yXAhcAv3PGvwSMB54BBjnvwfNAt+kiIiV142578MYYY4wj/xx8MLqLFZEoEVkL7AaW4HnOy0FVPeHMkgXkd8PnArYDONMPAeeUlG8NvDHGGON/9UVklddrXOEZVNWtqp2AeKArkOTPAqyBL8HiRQvpmNKWlKRWTJ400TIt0zLL6NChg9wweiS9unbg/G4dWZW5HIAXZjxFr64d6PObTjz0wP0hr9MyK1dmsPjpHPze/L5PnNeM4tanqhvOZpEAACAASURBVAfxPIL9N0AdEck/fR4PZDvvs4GmnvokGjgbT18txVPVKvnq0iVVj+Vpsa8jx09oi8RE3fD9Fj109Bft0KGjfvX1+hKXKe1lmZYZaZnZB34p8jV05NU6eeozmn3gF92667Bu2LZL33xvkfY8v4/+sPMnzT7wi369cXuRy1aWbbdM/2YCq0LdLqgqTdt20OmfbfX5Vdr2AA2AOs77GsCnwADgLWCkM/5Z4Gbn/S3As877kcCbpW2L7cEXY2VmJi1btqJFYiIxMTEMGzGSeXPnWKZlWmYpfjp0iBVffMqoa64DICYmhrPPrsPLL8zgljvvpXr16gDUb9AwpHVaZuXKjEBNgA9F5Bs83Z0vUdV5wH3w/9k78/goivQPPy+EKAiIKCAk3DfhDpcHKuoKyKGioHgAouJ9n7ve14LiBYuu4q7r+VNkV0VuvEU8EBBRUCAISIKuB65yB8L7+6NqoI05ZjIzmUnyPvn0J9M11d9+q7un36q3qqu5RkSycH3s//T5/wkc7NOvAW4qbgfm4Ath48Yc0tMb7l1PS0snJyeniC1M0zRNE+Dbb9dx8CF1uPrSCzjhqB5cd8VFbNu6lW+yVrPwowUMPP5ITh1wPEuXLEqonaZZtjRLC6F0HpNT1WWq2kVVO6pqe/Vvb1XVb1S1h6q2UNWhqrrTp+/w6y38998Ut4+EOngRyRORpSLypYhMFZFqge9OFhEVkTaBtCYist1vs0JEnhWRKiLS16ctFZEtIrLSf342MSUzjIpL3u7dfPH5Z4wYPYZ57y+kWrVqTHpkPHm7d/O/XzYx/Y353HLXWC4698xQqNIwkocYjKBPlpnwEt2C366qnVW1PZDL718DOxz4wP8PskbdqMMOuAEIw1R1rtfpDCwCzvLrI0pqWIMGaWRnb9i7npOTTVpaWhFbmKZpmiZA/QZp1G+QTtduPQAYMHgIX3z+GfXT0ug/6GREhC6Z3alUqRKbfv4pYXaaZtnSLE0qiUS9JAOJdvBB5gMtAESkOnAkbuaeMwrKrKp5wEL2PSMYU7p1705W1mrWrV1Lbm4uU6e8xICBg03TNE2zGOrWO5QGaelkrV4JwAfvv0Or1m3pe+JgPpz/HgBrslaRm7uL2gcfkjA7TbNsaRqRkxQz2fkh//2BOT7pJGCOqq4SkZ9FJFNVF+fbZn+gJ3BlPGxKSUnh4QmTGDSgL3l5eYwcNZp2GRmmaZqmGQZ33/8wl48Zxa7cXBo1acpDjz5JtWoHcO1lYzj2sC5USU3lkb//o0Rv7Ur2sptmfDRLi1AffHlAEtkHJiJ5wBd+dT5wrarmisgMYIKqviEiVwCNVPU6P7n+V8BKoCkwU1XPzKf5LnCdqv5hBI+faGAMQMNGjTJXrVkfn4IZRgXBpqo1YkXVKrJYVbsl2o4mbTvqrc/MiFrn/J6NE16eRLfgt/t+872ISG3cXLwdRESByoCKyPU+yxpV7SwihwALRGSwqr4ezs7UTTQwGSAzs5uN7jEMwzDKLcnUBx/iNOA5VW2sqk1UtSGwFugdzKSqP+GeAyz5dFiGYRiGkY9Septc3ElGBz8ceDVf2n/442h6gNeAaiLSu4DvDMMwDCMiBOcYo12SgYSG6FW1egFpfQpImxhYbR9IV6BTvrzHxNBEwzAMoyIhlGjwZzKSLBUNwzAMwzBiSKIH2RmGYRhGUlE+2u/m4A3DMAxjLwJJMxNdtJiDNwzDMIwA5cO9Wx+8YRiGYZRLrAVvGEaJicesczmbtsdcM6121ZhrGuWXchKhNwdvGIZhGPsQe0zOMAzDMIzkxRx8EcybO4eOGa3JaNOC8fePi1pvw4YN9D2+D106tqNrpwwmTZwQAytjb2dF1iwr5+jC80fTqEFdMju3Lz5zAjUhdmV/evKj9D+qG/2OyuRfT0wC4KvlyzjtxGM48ejuXHD2qWze/FvC7YynZrKfo3hrlgblaSa7ZLEj6cjLy+OqKy5l2vTZfLZsBVNfepGvVqyISjMlJYVx9z/IZ8tW8N4HH/PE449GrRkPOyuyZlk5R+eMHMW0GXOKz5hgzViVfdVXy5ny/L94Zc77zHj7E955Yzbr1q7hL9dcwvW33M2s9z7lhBMH849HH06onfHWTOZzFG/N0kREol6SAXPwhfDpwoU0b96Cps2akZqaytDTz2DG9GlRadavX58uXbsCUKNGDdq0acvGjTlJZ2dF1iwr5+jI3kdRu3btqDRKQzNWZc9avZJOXbtRtVo1UlJS6HH4kcybOY21a7LocdiRABxx9HHMmVmy41pWrs9kPkfx1ixNJAZLMmAOvhA2bswhPb3h3vW0tHRycqK70QdZv24dS5d+RvcePaPSiYedFVkzSDKfo7JCrMreqk07Fn3yIb9s+pnt27bx7ptz+S4nm5at2/Lm7OkAzJ7+Ct/nZCfUznhrxoOKXPbyTqk7eBHJE5GlIvKliEwVkWoFpE8XkVqBbTJE5G0RWSkiq0XkVvExEBEZJSJ7RKRjIP+XItKktMsWLlu2bGH4sFMZ/+Aj1KxZM9HmGAVg5yi5aNGqDWMuu4ZRpw9i9PCTaNe+I5UrV2bcI4/z/NNPctKfDmfrls1USY39Y3tGBUMsRB8N21W1s6q2B3KBiwpI3wRcCiAiVYHXgXGq2hr39rjDgUsCmtnAzbE0skGDNLKzN+xdz8nJJi0tLWrdXbt2MXzYqZw+/CxOPmVI1HrxsLMia0LZOEdlhViWfdhZo5j2xoe8OO0Nah5YiybNW9C8ZWueeXk60974kEGnDKNR46YJtzOemvGgIpe9IGyQXeyYD7QoIP0jIHQ1nAksUNV5AKq6DbgMuCmQfwaQISKtY2VYt+7dycpazbq1a8nNzWXqlJcYMHBwVJqqykUXnEfrNm258uprktbOiqxZVs5RWSGWZf/5xx8A2Ji9gXmzXmfwkNP3pu3Zs4dHH76P4SPPT7id8dSMBxW57OWdhDl4EUkB+gNf5EuvDByHa7UDZACLg3lUdQ1QXURCsdM9wP3AX2JlX0pKCg9PmMSgAX3p3KEtpw4dRruMjKg0P1ywgP974Tnee+dtemZ2pmdmZ+bMnpV0dlZkzbJyjkacPZxjeh/GqpUrad4knaef+mdUevHSjGXZLz3vTPr27sqYc07jjrEPU/PAWkx/dSrHH9aRE47oTN169Tlt+IiE2xlPzWQ/R/HULE3KS4heVLV0dyiSxz6nPh+4VlVzA+lpwFdAH1XNE5GHgPWqOiGfzi9AY2AI0A24ClgO9AOmAwNVdV2+bcYAYwAaNmqUuWrN+vgU0jCMEmNT1VZMqlaRxaraLdF2tMjopA+8ODdqnVM61U94eRLZB99ZVS9X1dxgOs5pC74PHlgBZAYFRKQZsEVV985qoaq7gQeBGwvbsapOVtVuqtqtziF1YlgkwzAMo7wgEv2SDCS6D/4P+D72K4BrfRj/BeBIETke9g66m4gLyefnaeB4wLy3YRiGUaFJOgcPoKqfAcuA4aq6HTgJuEVEVuLC+J8CkwrYLhfn/OuWormGYRhGOcGNopeol2Sg1N8mp6rVw0lX1UGBz18AxxSy3dO4lntofSLOyRuGYRhGxCRLiD1akrIFbxiGYRhGdNj74A3DMAxjL4IkSYg9WszBG4ZhGEaA8hKiNwdvGIZhGJ7QILvygPXBG4ZhGEY5xFrwMSQeswImy5SHRumyZ0/sr6VKlcrGtRSPWed++G1nzDXr1twv5ppGEpBEE9VEizl4wzAMwwhQXhy8hegNwzAMoxxiDr4I5s2dQ8eM1mS0acH4+8fFTDcvL49e3bsy5ORBxWcOg3jYaZrJrblq5Up6de+ydzn0kAOZNPGRpLMzmTXXrF5F/6N77F0yGtfhn4//jRVfLuPkvkdzwpGZjD5zCJt/+614sTjaaZqlj8TgLylQ1Qq5dO2aqdt3aaHLlh27tWmzZrpi5Rr9detO7dChoy75fHmR22zL3RPWMu7+B3TY6cO134kDis1b1P5KaqdpJr/m1p17wl5+27ZL69arp1+tWltkvrJS9nhorv95R5HLNz9s1Tp16+mCpSu1Y+dMnfL6PF3/8w69f+Ljevm1NxW4TVkpe1nRBBYl2i+oKq0yOumbX/0Y9ZIM5bEWfCF8unAhzZu3oGmzZqSmpjL09DOYMX1a1LrZ2dnMmT2LUaPPi4GV8bHTNJNfM8g7b79Fs2bNadS4cVQ6ZaXs8dBc8P7bNGrSlPSGjVm7ZjU9D+8NQO9jjmP29NeSxs6KrFmalEYLXkQaisg7IrJCRJaLyJU+vbaIvCEiq/3/g3y6iMhEEckSkWUi0rW4fZiDL4SNG3NIT2+4dz0tLZ2cnJyodW+49mruGXsflSrF5tDHw07TTH7NIP+e+hJDh50RtU5ZKXs8NF9/ZSqDh5wOQMs27Zg3azoAM6e9wnc52UljZ0XWLIfsBq5V1XZAL+BSEWkH3AS8paotgbf8OkB/oKVfxgB/L24HcXPwIpInIktF5EsRmSoi1QpIny4itQLbZIjI2yKy0tdebhX/nJiIjBKRH/22K0TkgkD6HhHpGND5UkSaxKtsJWXWzBnUqVuHrl0zi89sGGGQm5vLrBnTOeXUoYk2pcySm5vLm3NmMuCkIQCMn/gEzz31BAOOPYytWzZTJTU1wRYapU1pvA9eVb9T1SX+82bgKyAN9/bUZ3y2Z4CT/eeTgGfV8TFQS0TqF7WPeLbgt6tqZ1VtD+QCFxWQvgm4FPa+5/11YJyqtgY6AYcDlwQ0p6hqZ9yb5f4qIvV8ejZwcyyNb9AgjezsDXvXc3KySUtLi0rz4w8XMHPGdNq0bMqIs4fz3jtvM3rkOUlnp2kmv2aIeXNm06lzV+rVq1d85mIoK2WPtea7b86lfcfO1KnrjmGLVq15/j8zmfn2RwwecjqNmzRLCjsrumZpEqMQ/SEisiiwjCl0f65B2gX4BKinqt/5r74HQj/uNGBDYLNsn1YopRWinw+0KCD9I/YZeCawQFXnAajqNuAy9oUn9qKqPwBrgFCn4wwgQ0Rax8rgbt27k5W1mnVr15Kbm8vUKS8xYODgqDTvuncsWWs38PXqtTz7/Isc3edYnnrmuaSz0zSTXzPE1JdfYujp0YfnoeyUPdaar7/yMoOHDNu7/tOPPwCwZ88e/vbgWM469/yksLOia5YWAlSS6BfgJ1XtFlgmF7g/kerAf4CrVPV3j2yoqgIlnvUq7hPdiEgKru9gTr70ysBxwD99UgawOJhHVdeISHURqZlv22ZAMyALaAfsAe4H/gKMjIXdKSkpPDxhEoMG9CUvL4+Ro0bTLiMjFtIxJR52mmbyawJs3bqVt996g4mPPh61FpSdssdSc9vWrcx/9y3++tCkvWmvv/Iyz/7THdN+A05m2Jklu6Uke9nLmmZ5RESq4Jz7C6r6ik/+r4jUV9XvfAj+B5+eAzQMbJ7u0wrXdxWE2CMiecAXfnU+bjBBbiA9Ddfn0EdV80TkIWC9qk7Ip/MLrqU+BBiPK9BOXCj/VREZBXQDrgKWA/2A6cBAVV2XT2sMbnACDRs1yly1Zn1MyxyPY2lT1VZMKvJUtfHApqpNfqpWkcWq2i3RdrRp30WffOXtqHWOal27yPL48WXPAJtU9apA+njgZ1UdJyI3AbVV9QYRGYCLap8I9AQmqmqPomyIZwt+u+8vLzDdD7qbi+uDnwisAI4KZvQt9S2q+pt3dFNU9bKCdqaqu0XkQeDGwgzyIZLJAJmZ3eJTszEMwzDKLqU3F/0RwDnAFyKy1Kf9BRgHvCwi5wHrgVD/0Sycc88CtgHnFreDhM1Fr6rbROQK4DUReQx4AfiLiByvqm/6QXcTcaH3cHkauAGoEXODDcMwDCNGqOoHUOgD88cVkF/xg9LDJaHPwavqZ8AyYLiqbsc9BnCLiKzEhfE/BSYVIZFfLxdXKagbB3MNwzCMCoDEYEkG4taCV9Xq4aSr6qDA5y9wj8AVtN3TuBZ6kemqOhHn5A3DMAwjItwo+mRx0dFhr4s1DMMwjADlw73bVLWGYRiGUS6xFrxhGIZhBCknTXhz8IZhGIYRIGne5x4lFqI3DMMwjHKIteBjiM06Z8SKijzrXDyIx6xz97y5KuaatxzfKuaaRuSUl1u5OXjDMAzDCFBO/Ls5eMMwDMP4HeXEw1sffBFceP5oGjWoS2bn9jHTnDd3Dh0zWpPRpgXj7x9nmqZpmmVcc3fuTp668jSevGQwT1w4gPeec/NszXj4Lzx5yWCevHgQ/7nnCnK3bwXg1x828vyN5/CPS0/myYsHkbXwvVKxszxoGpERt7fJJTuZmd10wSeLiszzwfz3OeCA6pw/egSLl34Z9T7z8vLo0K4VM2e/QVp6Okf26s4zz79I23btTNM0TbOMaYb64FWVXTu2kVr1APJ27+LZ687khAtv5pBGLdjvADdx5xuTx3JArYM5fNgYZk64lUObtyVz4Jn8uD6LKbeN4bJn3NvLiuuDT5ayx0MzWd4m165DF3329cgqXQXRvdmBCS+PteCL4MjeR1G7du2Y6X26cCHNm7egabNmpKamMvT0M5gxfZppmqZplmFNESG16gEA7Nm9m7zdu0Fkr3NXVXbv3PG7/Du3bQFg57bNVD84/FdnJFvZS1Oz1PBvk4t2SQbMwZciGzfmkJ7ecO96Wlo6OTk5pmmaplnGNffk5fHkpSfx8PDDadblcNLadAJg+kN/ZsKZR/Bz9jd0H3wOAL3Pvowv35nOxLOPYsptY+h78S2lZmdZ1jQiJ64OXkTyRGSpiHwpIlNFpJqINBGRL/Plu0NErvOfnxaRHBHZz68fIiLrAnlbicgsEVktIktE5GUROd3vZ6mIbBGRlf7zs/Esn2EYBkClypW54NFpXPHce2xctYwf1rnw/aBrxnLF8/M5uGFzVrw/C4AV786k4/GncMXz73P6XZN5ffwN6J49iTTfyEd5eZtcvFvw21W1s6q2B3KBi8LcLg8YnT9RRPYHZgJ/V9WWqtoVeAxY7vfTGVgEnOXXR8SmGLGhQYM0srM37F3PyckmLS3NNE3TNMuJ5v7Va9K4Y0++WTR/b1qlypXJOHoAXy+YB8DSuf+m3VH9AUhv24Xdu3ay7bdfStXOsqhZqpQTD1+aIfr5QIsw8z4CXC0i+R/jOxP4SFWnhxJU9V1VjX4EXCnQrXt3srJWs27tWnJzc5k65SUGDBxsmqZpmmVYc+v/NrFjy28A7Nq5g7WffUjt9KZs2rgecH3wqz5+m4PTmwFQs2591i79CICfvl3D7tydVDswvLE+yVb20tQsPSQmf8lAqTwH7x11f2BOmJt8C3wAnANMD6S3BxbH1rrCGXH2cOa/9y4//fQTzZukc+ttdzJq9Hkl1ktJSeHhCZMYNKAveXl5jBw1mnYZGVHZaJqmaZqJ1dzyyw9Mf+AmdE8eqkrb3v1o2eMYnr3+THZu2wqq1G3amv6X3QnA8effxKyJt7Dw1adBhEHXjAt7FsxkK3tpahqRE9fH5EQkD/jCr84HrgXqAzN92D6U7w5gs6o+KCJPAzOAz4FpwDHAQlVtIiIPAetVdUIR+3wXuE5V//AMnIiMAcYANGzUKHPVmvXRFtEwjAqKTVUbW5LmMbmOXfX/ZkT/mFyXxjUTXp54t+C3+37xvYjIz8BB+fLVBtYGE1R1tYgsBYYFkpcDR5fUGFWdDEwG9xx8SXUMwzCM8kkSdaFHTak/JqeqW4DvRORYABGpDfTDheTzcy9wXWD9/4DDRWRAKEFEjhKR2E01ZxiGYRjlgEQ9Bz8CuNW30N8G7lTVNfkzqepyYElgfTswELjcPya3ArgE+LF0zDYMwzDKPeVkFH1cQ/SqWr2Q9BVAn0K+G5VvfUi+9a9xLf7C9nlMpHYahmEYRohkGQUfLfY2OcMwDMMIkCxTzUaLTVVrGIZhGOUQa8EbhmEYRoBy0oA3B28YhmEYe0miQXLRYg7eMAzDMALYIDujVIjHTIPhTotpGEbhxGPWuS07dsdcs/r+dpuvqNiZNwzDMAyPUH5G0ZuDNwzDMIwA5cS/22NyRTFv7hw6ZrQmo00Lxt8/Lmk1AfLy8ujVvStDTh4UtdaGDRvoe3wfunRsR9dOGUyaWOi7fSIiHmW/8PzRNGpQl8zOsZutuKycd9OsOJp5eXn0OaIbZ5520u/S/3z9VTQ+tFa0ZiZ12Y2SYw6+EPLy8rjqikuZNn02ny1bwdSXXuSrFSuSTjPEo3+bQJs2bWOilZKSwrj7H+SzZSt474OPeeLxR5O27OeMHMW0GeG+hbh4ysp5N82KpTn5sYm0av373/fSJYv49X+/RGVjrO2Mp2apUk6mqjUHXwifLlxI8+YtaNqsGampqQw9/QxmTJ+WdJoA2dnZzJk9K6p31QepX78+Xbp2BaBGjRq0adOWjRtzotKMV9mP7H0UtWvXjlonRFk576ZZcTQ35mTzxtzZnD1y9N60vLw87rjlJm67O/qWcTKXPVFIDP6SAXPwhbBxYw7p6Q33rqelpZOTE52Ti4cmwA3XXs09Y++jUqXYn87169axdOlndO/RMyqdeJU91pSV826aFUfz5huv5fa7x/7u9/2PJx6l34kDOfTQ+lHZGEs7461ZmohEvyQDpebgReRmEVkuIstEZKmI9BSRKiIyzr8ZbomIfCQi/X3+A0XkWRHJEpE1/vOB/rsmIqIicnlAf5KIjPKfnxaR00qrbIlk1swZ1Klbh65dM2OuvWXLFoYPO5XxDz5CzZo1Y65vGEbRzJs9kzp16tCpy77f9/ffbeT1V//D+RddlkDLjLJAqYyiF5HDcK957aqqO0XkECAVuBuoD7T36fWAo/1m/wS+VNURXuNO4B/AUP/9D8CVIvKEqubG2uYGDdLIzt6wdz0nJ5u0tLSk0/z4wwXMnDGduXNms2PHDjb/9hujR57DU888F5Xurl27GD7sVE4ffhYnnzKk+A2KIR5ljwdl5bybZsXQ/OTjD5kzawZvzpvDjh072LL5N47s0YnU1P3o0akNANu3baN7pzZ8+vnXCbOzNDRLkyRpgEdNabXg6wM/qepOAFX9CfgfcAFweSD9v6r6soi0ADJxFYAQdwHdRKS5X/8ReAsYGQ+Du3XvTlbWatatXUtubi5Tp7zEgIGDk07zrnvHkrV2A1+vXsuzz7/I0X2Ojdq5qyoXXXAerdu05cqrr4lKK0Q8yh4Pysp5N82KoXnrnfeybOU6lizP4smnX+DIo/qQteFHVqzJZsnyLJYsz6JqtWoldu6xsrM0NEuVcjLIrrSeg58H3CYiq4A3gSnAL8C3qvpbAfnbAUtVNS+UoKp5IrIUyACW+eT7gNki8lSsDU5JSeHhCZMYNKAveXl5jBw1mnYZGUmnGQ8+XLCA/3vhOdq370DPzM4A3HnPX+nX/8QSa8ar7CPOHs78997lp59+onmTdG697c6oBhuWlfNumhVTMx5U5LKXdyQeU6EWuCORykBvoA9wIfBX4FxV7VJA3sH+u1Pypb8K/Avn4GeoansReRZ4A+gJLFLVp0Xkaf/9v/NtPwYYA9CwUaPMVWvWx7iUscemqjWMikNFnqq2ahVZrKrdEm1Hh05d9ZV5C6LWaXVotSLL4xumA4EfVLW9T6uNawA3AdYBw1T1F3E37QnAicA2YJSqLinOhlIbZKeqear6rqreDlwGDAIaiUhBo7dWAJ1FZK99/nNn/12QvwI3EkZQRFUnq2o3Ve1W55A6JS2KYRiGUV6JwQj6MNtQTwP98qXdBLylqi1xXdA3+fT+QEu/jAH+Hs4OSsXBi0hrEWkZSOoMrMQNpJsgIqk+Xx0RGaqqWcBnwC2BbW4Blvjv9qKqX+OcfvRTuBmGYRgVntLoglfV94FN+ZJPAp7xn58BTg6kP6uOj4FaIlLsM5Kl1YKvDjwjIitEZBmuj/0OnNP+EVghIl8CM4BQn/x5QCv/iNwaoJVPK4h7gfTAegqwM+alMAzDMIz4UU9Vv/Ofvwfq+c9pwIZAvmyfViSl0jmjqouBwwv5+ga/5N/mF+DsQvTWAe0D65/jKys+lN8WWBOV0YZhGEbFJDbDlA4RkUWB9cmqOjncjVVVRSSqQVhlY/RFmIhIA9wo/XdUtQxNfGwYhmEkBzGbavanEgwa/K+I1FfV73wI/gefngM0DORL92lFUq6mqlXVjaraTlUvTbQthmEYhhEhr7NvbpeRwLRA+ghx9AJ+DYTyC6VcteANwzAMI1pK40liEXkROAYXys8GbgfGAS+LyHnAemCYzz4L94hcFu4xuXPD2Yc5eMMwDMPwlNZEdKo6vJCvjisgrwIRR6bNwRuGYRhGkHIyF5g5+Bhis84ZscKupYpJPGade+T92D9QdNVRzYvPZCQcc/CGYRiGESBGo+gTjjl4wzAMwwhQXoJd5eoxuVgzb+4cOma0JqNNC8bfPy5munl5efTq3pUhJ8dmdt142Gmaya25Y8cOeh/ek56Zncns1J6777w9BlaWjbKbZuSau3J38uglQ5hwwUAeHt2PN55+BIAPX3uW8eccy5+Pa8HWX/fNmrp98688d9vFTDh/AI9eMoTv164qFTuNGKOqFXLp2jVTt+/SQpctO3Zr02bNdMXKNfrr1p3aoUNHXfL58iK32Za7J6xl3P0P6LDTh2u/EwcUm7eo/ZXUTtNMfs3iroutO/P0h02/6bbcPfrr1p3arXsPfXf+h3YtmeYflrFvZelf31ytd8z4XMe+laX3zP1K09t00ov/NlUvf3ya3vDCu1qrXpre8spCHftWlo59K0t7Dztfjxt5hY59K0uv/tdcbd7lsL3fjX0rKy524t4GmnDf0KFTV/325x1RL8lQHmvBF8KnCxfSvHkLmjZrRmpqKkNPP4MZ06cVv2ExZGdnM2f2rKjeWR4kHnaaZvJrigjVq1cHYNeuXezatSvquGJZKbtpRq4pIuxX9QAA8nbvZs9ud700aJnBQYemVCoTJAAAIABJREFU/yH/D+uzaN75MADqNmrOL99ns3nTT3G3MykovbfJxR1z8IWwcWMO6en7ZgZMS0snJ6fYmQGL5YZrr+aesfdRqVJsDn087DTN5NcE19XTs1sXGqfV47jjjqdHj55R6ZWVsptmyTT35OUxccwg7j21Jy0yj6RR286F5q3frA3LP5gLwIavP+d//93Ibz99Xyp2Jgel8T65+JMQBy8i74hI33xpV4nIbBHZLiJLA8sI//06EflCRJaJyHsi0jiwbZ7P+7mILBGRwl5sk1BmzZxBnbp16No1M9GmGOWAypUr88miz1i9dgOLFn3K8i+/TLRJRhJTqXJlrpg8nZumfED2158X2a9+9PAL2bFlMxPHDOLDV5+lfst2SIwaJUbpkahR9C8CZwBzA2ln4N4q11BVC6ta9lHVn0TkTtyrZi/w6dtD2/iKw1jg6GgMbNAgjezsfW/ny8nJJi2t2LfzFcnHHy5g5ozpzJ0zmx07drD5t98YPfIcnnrmuaSy0zSTXzNIrVq1OOroY3hj3hwy2rcvfoNCKCtlN83oNKtWr0mzzr1Y9en7HNq0VYF59j+gBqfdcB/gxmndf9Yx1K7fsMC88bIzUQjJE2KPlkRVyf4NDBCRVAARaQI04Pfvuy2Kjyj8Xbg1gV+itI9u3buTlbWadWvXkpuby9QpLzFg4OCoNO+6dyxZazfw9eq1PPv8ixzd59ionHu87DTN5Nf88ccf+d///gfA9u3befutN2nVuk3S2WmayaG55X8/s33LbwDs2rmDrMULqNOwWaH5t2/5jd27cgH4dNYUmnbszv4H1Ii7nclC+QjQJ6gFr6qbRGQh0B/3tpwzgJcBBZqLyNJA9stVdX4+iX7Aa4H1qn6b/YH6wLHR2piSksLDEyYxaEBf8vLyGDlqNO0yMqKVjTnxsNM0k1/z++++44LzRrEnL489e/Yw5LShnDhgYNLZaZrJobn55x+Zev/1aN4eVPfQ4egTaXvYsSx45RnenzKZLZt+YsIFA2nd42hOvW4sP6zPYup9NyAi1GvSklOvG1sqdiYL5aUFL6qxnxIzrB2LnAUMVNXh3jmfB/wMzFDVP8QZRWQdsBmoDWwBuqnqZv/dFlWt7j8fBvwDaK/5CiciY4AxAA0bNcpctWZ9TMsUj2Np04tWTOxaMmJFWZmqtmoVWayRvz895nTqkqlz3v0oap0GtfZLeHkSOWpiGnCciHQFqqnq4jC26QM0BpYCdxaUQVU/Ag4B6hTw3WRV7aaq3eoc8oevDcMwDAOJwV8ykDAHr6pbgHeAp3CD7sLdbjdwFTBCRGrn/15E2gCVcdEAwzAMw4iMctIJn+jnHl4EOvF7B98832NyV+TfSFW/89uE3o9bNZQfmAKMVNW8eBtvGIZhGMlKQl82o6qvEajrqOo6oGoheZvkW7888LlyfCw0DMMwKhpJ0gCPGnubnGEYhmF4kmmq2WgxB28YhmEYAZJlkFy0JLoP3jAMwzCMOGAteMMwDMMIUj4a8ObgDcMwDCNIOfHv5uBjic0UZsQKu5aMWBGPWeeMsoE5eMMwDMMIUF7q1+bgDcMwDGMvyTPVbLTYKPoimDd3Dh0zWpPRpgXj7x9nmqZpmqZpmmGwYcMG+h7fhy4d29G1UwaTJk6IgZWlQ+h98NEuSYGqVsila9dM3b5LC1227NitTZs10xUr1+ivW3dqhw4ddcnny4vcprjFNE3TNE2zImh+8+1G/fCTxbp9l+oPm37TFi1bFqsJLEq0X1BVOnfJ1E1bd0e9JEN5rAVfCJ8uXEjz5i1o2qwZqampDD39DGZMn2aapmmapmmaxVC/fn26dO0KQI0aNWjTpi0bN+ZEpWlEjjn4Qti4MYf09IZ719PS0snJie4CNU3TNE3TrAiaQdavW8fSpZ/RvUfPmGnGm/ISoi8XDl5EGorI2tDrY0XkIL/eJLGWGYZhVFy2bNnC8GGnMv7BR6hZs2aizalwlAsHr6obgL8DodEh44DJ6t5OVyIaNEgjO3vD3vWcnGzS0tKiMdM0TdM0TbNCaALs2rWL4cNO5fThZ3HyKUOi1itNJAZ/SUGiBwHEagGqAMuAq4DlQJVoBtlt3r5LmzRtql+t+mbvwJPFS7+MauCJaZqmaZpmRdDclrtHzzzrHL308ivD3oYkGJSmqnTumqm/bs+LekmG8pSb5+BVdZeIXA/MAU5Q1V3R6KWkpPDwhEkMGtCXvLw8Ro4aTbuMjKhsNE3TNE3TrAiaHy5YwP+98Bzt23egZ2ZnAO6856/0639iVLqlgVB+pqoV3/otF4jII8AwYLyqPlzA92OAMQANGzXKXLVmfSlbaBiGYRRE1SqyWFW7JdqOrpnd9L0FC6PWqVm1csLLUy764AFEpDPwJ6AXcLWI1M+fR1Unq2o3Ve1W55A6pW6jYRiGUQaQGCxJQLlw8OLezPF34CpV/RYYDzyQWKsMwzCMskh5GWRXLhw8cAHwraq+4dcfA9qKyNEJtMkwDMMwEka5GGSnqpOByYH1PKBr4iwyDMMwyirJMlFNtJQLB28YhmEYsaKc+PdyE6I3DMMwjNhQSoPsRKSfiKwUkSwRuSm2hTAHbxiGYRiljohUBh4F+gPtgOEi0i6W+zAHbxiGYRgBSmkUfQ8gS1W/UdVc4CXgpFiWw/rgDcMwDMMjlNoguzRgQ2A9G4jpK/cqrINfsmTxT1WrSLhT2R0C/BRjE0yz4mmWBRtN0zQTpdk4xvsuEUuWLJ5btYocEgOp/UVkUWB9sn/iq9SosA5eVcOeyk5EFsV6ykHTrHiaZcFG0zTNZNeMN6rar5R2lQM0DKyn+7SYYX3whmEYhlH6fAq0FJGmIpIKnAG8HssdVNgWvGEYhmEkClXdLSKXAXOBysBTqro8lvswBx8e8eg3Mc2Kp1kWbDRN00x2zXKDqs4CZsVLv1y9LtYwDMMwDIf1wRuGYRhGOcQcvGEYhmGUQ8zBh4F/3/ze/zHSrBxrzXghIt1EpJv/XDlGmgfFQqeYfVTy/6OyWUQGiMipsbEKRKRqrI5jEfuIxXO8cUNEjheRPyfajkjwI51jrdlDRJrGWjeWhH5HEeTvIyKHx8seI3zMwYdH/eBKtE5ZRA4G7hCRJqqqMa441BORhsXnjIizgQmw91W8USEiHYG7o9UpQl9EpC6wSEQOUdW8kjpUETkBuA/4MUa21QX+DPSJh5P3ZT8UeFdEop72UkQOE5G+MTAtqNkXGA+8E2Pdw0Xk4lhqBrTbAa+LSFoMNfvh5iKvESO9uiLSM1KHXIhWUxFpIiKVVXVPhJpHA+d4HfMxCcQOfjGIyP7AFBHpDBwGoNGPTGwC1AQuFZFGsXLyIjIA9xzlHBEZF20rOWDT9UC2iJwcrY2eX4Ej/A0k5hEMdfwAfAF8KCIHl8TJe0f0HPB3VX3fp5XYXhFpDBwO1AYG4Y5BTJ28L/v3OAd6m78mIsZXFOoAHwB3icgpsbDPH9P/A+5X1Y9joRlgD3CdiIyJsS6qugJXyfu7iDSIVs8fh4nADaq6LEa/g/OBi4Be0ThWEekPzMZVwl8LOPlwbfwQd39DVfeU1A4jeszBF4GIdMHNDXwJ8BDwjIjUjlZXVRfjHh/ZA1wVCyfvWwMPAKOAE3GVkSuitDNUkRFgJdArGj0RqSwiqaq6HvgG2N+XO2bXoYg0FJF/A6jqSGAe8GmkTt47xgeBd4HqInKE1yxR5U5EWuNumptV9TJgMzCUGDp5EWklIgeKiKjqM8A4YJyIDIxUy1cUfgTGAquAY0TknCjtG+j1PgN6ikiLWFbwfIXhbFzF+ZJYaIpI7dD5UdVzcPOFPxWNk/cO9ElgF/CT/02UuNEgIoeISHPc7/9bYDhwWEl+V/66/zNwJXAp8F/gYHDXRGHnS0SOE5HL/O9kNdAk/zGy1nzpYwe8aFKBTNwPpjVwnapuKomTF5FmInKhiNwuIqNwP4K/A7uBK6Nx8iJSAxjiNTd4B/oXoI2UoN/Qh/leFZHGIlLLv+noGeA0KWFftK8szQZe9KHjdKCfiFTKX8uP5qavqhuA1iIyy69fhnvONGwnL65P9FbgXFyLqCYwUERKVMHxzn0u8IiqvuWT78ZFMmLi5EWkGfA18BHwvLg+0LeAq4C7ReS4CLSqBVaX487VEiBTRM4qoX0HAacDF6nq8biw9F+AqPqfxfXlvykiV4pIM1X9yO9njIhcEKV2V+B74GURuUpEaqvqJcBS4PGSOHl/LYwHBgD34iqRR5b0/ItICnA1rjLfGLgHd12dQQRO3kdsagPTgXdVdS7QwOuME5GFIpJeRIU8FegK3ICrvLQFRovI4JC+teZLH3sOvghEpAouRNsC1wd9GPAa0Bt4SFV/CVOnDfBvYArQCHdza4trEdcDLsZNOjRBVb8toa3dgcFAVdyP/FqgrqpeWAKtQ3B9g3nATtxLEj7yN/c2wF+BneH+YEXkAFXdKiI9cBWltkAfoD0wA3dMNwFf+PByxPgwYl5gfb638Xi/PgkX2eiuqj8XVLHw+QbgojZP+soCItIWOBM329TrkYSW/bl/AVeRywHOVtVt/rtU4DbgQGAq8EFJboKB43s70AGoA7wJDMNFns7AvfTjTlUtcipM37q8DHjZRwEQkTuB/XAViCOAt1T1pQjsO1BVf/UOcpNPE1wUqzJwj6p+E2GZBddAGYlzlFtwx3c5rpKzDVeJujsSW/PtozUuCnIQ7jf7LpDh9zcRWAbcrKrZYer18NuvUtUFPu1S3CtCxwLvl2SMi7/GzsddY08Ba4HbcdfVS8BHEfxWhwKPATfhKp/vqOp9IvIocALQpjgbfQX5XtwxOwD3226Aq8y8EoMuTiNcVNWWwALUDnwehOt3+8ivj8eFVk+JQK8d8Alwar70F3B9xPux76YxFueUw9WuD7TC1dwrAc1xA8IW42rhoXwSpl4DoCVQ3683AW7EhWhvxFVyXgOaRGBjK2ASzmlWCqS3AP6D6/64Axe2fSx4/CPYRxPczfZa4KRA+uvAm4H1STgndVAhOn/yOif49cqB71rjHQbQO0y76voyjvDrj+G6DKoF8qR6zQmRHNfA9o1xlaQOuMrdJV7raFzr+Hzgn7juoO+A6sXo3YQLHW/AtQovwbU2zwWqASOAF4HTwrSvHy6SMBk4C9g/eD3iWntPAi0jLbvfPmTT3cBdQCdcK/Qu3G91Dy5qEIlme+BYnGPPxLWQH/bHc5T/LSz02u8Er5MiNE8EPsZFhNLyHYNL/HVxTDhagWu+X+j3AhyKq8yNxVWgq+DuKRNwldqitNKBWsABfn0IrrLwr3z5pgHNitAJ/r6vDm2Pu0+NLGpbW+KzJNyAZFr8DfkWnNPtgGuxp+Bamx/hQ3PAqWH+qMXn3xBI2y/w+QXgKv/5CFyr4KgwbR0AvA+8B7wBzMHd4NP8zfQRoGbIjgj13vR61f13R+GiDDP9Te0f4Wj6bWvgHOsD/niGokZNca2thn69FlCrhOetP64S8gGwzt/oHvDncxXwdCDvQ7iWbH6NE3BOrWPAvmuA1ECeVn77W3HjB4qyqQ6u5fynQNoBOCc/l987+f1wXSC3l/Ca/TPwCu7GXt3fXJ8CDgvk6wKkFaHTG6jhP48AVuDCrSNwLcLfcJW0Wrguq/phXlOf+PNzuy97C/9dlUC+l/w1khJmmXsBF+AiFL182khcReZ4v17J2z4RaBfB8eyHG28yA1dR7uWP3Xic8zzI56sPnBIqTzGaR/nrsFu+9GAF8mJ/rIqtPOLuSR/ifovzcRXbU3EV/EeA63AV6P1xEYjJFPJ79eX9Ancvep59FYYBuFD/gMA18QVQJ8zj2Bh4viS/Z1tityTcgGRZcLXYHriQ6SM4J3aw/64LsB7XKuiNG4CSGqZuDX/zfjX0gw5tiwulTwjkfQa4MQzNvv7m0wfnNA71P85VOCffAteieRJIL6HeczhnWSOQ7wB8v2mEx7YGzuE+hHPylXz6g8BxMTp/J/syX+fP0YM4p/GGvxFO9/muBP6Wb9sqfruluP72av5me10B+2kdzk0O1w/8nL8xHhJIr4pzdDP5vZO/CBemD+u6yrevOjin/jouYlQV1/f+ODA4TI3JwEb2VeouxUVVGuIqNmOADv67SmHa9D9gbGgbnAMe5dclX/5iKww+3wBchWO8P44bfVlTcK3rfwHDA/nDahH7vL1xg8q6+vW7/TGogqvcjMNFyJpEeH7OAy4NXWvB8vP7lvz5QKMwNZv762UJcByucn4/LrLwBu5JhQY+78dAzwI0+vvr/FjcvW+ityFk26m4d7k/ge+eiKDMtXCVxD/s15bSWxJuQDIsuFbPff5Gdg+uP69X4LtXfPr1uLDdIcXotca1chr59VTgWVyIKxjGOhfnMKvgnOcUXB9XUdodcQ7rKL8ebAU8720VnCO9BagXhd6zuHBnSv4bcjGaGcDEfGk1/E35AaCVT/s7rq+7JOesBa4VdzFwtE87BfgbMNKvV8NVzu4HTvRpVwNtC9A7ELgcF71YDZyT7/v6hOfYguf3DFxFcRSBSgGuZfWU31clf77OAtqHWfajgP750ur6sr0KNMM9incjrrJaLUzdR3Gt15CTvx7nQDL8etjXgM9/Ku6VmEP8+gvAIm/jff7cHBCBXktcK7J3IK0dbmR7KBIW6kIYGoGu+Gv8XFzLfUDgu9fYV9HvhXOCd+HGDoQbxXoYeKaIMoXbKq6Nc5xN/XoT3ADS2/x6GnAzbjDrHpzTboBztA3zae0PfIV7BDR4P7rV/25C18BgnJPvEOG5F9z9p0Ek29kS2yXhBiTLghsBmoWrAV+Fa22ejGvdhmrdDxb3Y/QX9oO4fsyX/I87FTfI6W/ADJ+vo7+Z9glsW2zrDeco/4Nr7YciAvv5/y1wfXl1/HqRYeQw9eaG+yNlX82/E64F8WAB+/oHMMmvNybCaIDfri0uWnEXrpX8H2Cm/24gruU6Gh/yJxD65feh4Za459L7sK8743pcP3zLQL5R/twV6YxwFbtHcE4mdCyOY5+TrxfIW5UIWkT59jMcNzakb770urg+9BsDx7fA8Qb++3rAgfnSnuD3Tv5KXCSnYwltPQkXGZmFqyy2wLVoJ+H6rw+MQKsV7pWa4Bxs6HfZDhdh64RzgmF1IRSgf6A/d//wdt+Dq5QHu2m6E14Epze+7x/Xp/8ccGTg+9Bv7aZgehF6GbguqBdxUYW7cFHHJriK4oOBvFUIjOUhX4ME9/upirvnfR2w80ncUwNzcBWzs3ENmv2Ks68Qm8PqcrElfkvCDUiWBdf6+wbfX4trbX5PBAPqAlp/8j/G5rgWwCP+ZhEaBPcpLiQ+0Of/Q7iuAM1DA5/3w7WGXgukib9BzaOIvtYS6oUbPg1qtsGFSyfw+zBkPVwLI+yWW759VMf1P57v10Mt4OnAHJ92Ki7kfCGFtLRwod4luNbkm7jnhzv5Y3Gl12uGC2MuJozWNW70+R5cBOhunHPoANyJ64MeQQnHGQTPS+B6Xcu+yEQwrPqvcM6VP+fD8ZWbwHeP+WO8v18fQxgDpPyxuoZ8rT1cRflH4Ix8tobt3H3+VjiH1Cl47fr/z+CjGkQWlg/25/f0aaNwlY+v8u8nAt0jcIMaQ9fp/binT44K5DkdF9FoUoxWU1xre7i//lv7a+s+nINvgvudPlHY9RJYDw32a+3Xu+EaNm/h7lW1cE/6XI+LVlgLvAwvCTcgGRb29S139DfOe3Bh2ucCeSINT74G3Oo/j/A3/Q9wTue70I05TK02OMfxMDAmYPNknCMK3TBH4frfapamXj7NR4DRPq09ruU7KZBvOC4qULWE56omrjUYar0FW+T/wVfIvO0FDq7CDSz6GB/a92m345x8e79+ub/xraaAkH4+vabsixacgxs8ONR//jvOWazCPcZ1HhE4oMA++uJGRb+MC5vWxkUH1gL9AvmG4rp6Cq1A4cdl4FpoTwOn8ceW/Av4CmiY9gmuMrcJF7l6DleZC0UCTsU5sxGR/KZwIeZU9o3wHodr9TYMavhrd3gkv1UK788Xf/08FckxKED/SH8NDcdFr8b5Y/Nv3JMjywkj9O1teTR4vePGRrzMvjEOzXD95IVeq/4a+gQf+Qlcsx1wYfwr8x3TErXcbUmeJeEGJMvCvoFf1+NGU/fDjZq9LUKd0I+jO64V1wn4Ehcy7odzJIcF84ahmY6rHNyIq6k/629OjXBdCS/4m/WiMG8YMdUrQPMNf3M8BRcCvw/nUG8APifMvuZ8+jUCx3YOgcgKPhSI69+/pRid2riKSCh6sn/guzuANbhWUi1ci7xI5+63Ox43oCw0wjo0GUqoldQc17J9m3xh9TDLPhjXghvqbXwAN9aiBW6A1GJcyPY+XPi20OOLc7qTgCv8emgw4GkEogu4ikmx0St+H53pj3va5FBcmPshnONt4r8fiBv1XYPwnHsoevKiL1t1XHfKk7i+5u4+3zn+vIXd3UPR/flX+/UROIcc7iOBx+F+S0ewrwLS05+7M/16M1zf9OkU82gg++5Jo4HH/efKgfRGXjs0+LHQLjl/jW4Gjg1ck28Cnf16N1x05PpIr09bkndJuAHJsuBq7bVxTmourrZdiXyDUyLQq+t1tgMXBtJL2nJ9CNcyS8G1CF7D1djb48YNbCCyx4FiqleA5pm4FvX7uMFUD+BaEK1KUPZWuJDkmbjBbrfiWrPN8uW7wN88haK7Owb4m3to8FTw0cV32DeKuthBdYHt+uGcTMjJX4mrzBxWQN5IBizW9jb1CqQ1xEUY/o2riPTEPSb253CcBvteHhTqez0d15I/D+eABuJaeuGE5VPyrb/MvsjVlcBWXFfIONxz3mFd/7g5KBbgWsGn4MLFofkJ+uEqz2txDnUJEVYaiXF/vteYhKs8zsY53stxkYtuuHEMYVUUvF4zv+/GuP73NcAxgXNY1X9+DmhcjFYj3G/xTdyTDE1wIfnrg9e5t3MRRYzbsKVsLQk3IJkWnGM6xH8ej3cAUeh1x7VoQhPHhO0wAhqhVmsqrjVxqL9RfoNryUzBDWgrcvR9vPSK0VzrNV/GhUAjrtzgBgQtw43wrevTmuKc2z3458xxg5rWEHjuvBjd/vzeIYdu8NOIcMRwEZqX+vVeJdHzGnVwjq45v28tN8J1f4T1mCGuxRqKKAjOgT6GewpBvO0P4JzTG+EcA9xYkxdxIfOTfVovXDThJH9NHYkbzHUF4T2yWQk3AOx7YEog/Ub8PAHsc0gN/bUW9uRQAb2Y9ef7Y9scN2ZlIi4KeBnuyYj5uEG3X+Gc/5Aw9NrgxulcDhweuJZe4PfdSp1wlZviKnXtcY9QZuDua3uAy/Mdy6O8/RE/pmlL8i4JNyDZFvY5q4j7SQvQqoILVQ6jBM49aBP7Zjx7wd+YQjfUVkRY4461XpiaJZmhLgVXObisgO+a41qFs3Ch788I85nvgEZ+hzwC13KN2GEUoBnq37ySMGe+y6fTiH0TzzzLvqhC8ImA8cBjYWgd7G/qP3hHcRH7WvK3+7RQN0cdwhgIiGtFL/RO6G7cgMquuIjC27hpjo8Nnsswyx2aI6IpriV9s19/EtcKfhvXrTSUYlquBWjHvD+ffY/YPuhtrod7EuFmXEs5BeiM69ufSzERLG/jl/gBiYH0Zv5Yr/Ln/VZc/36xFQa//c24brJGuFb8i4HvzsUNqix2cK4tZWtJuAHlfcG14g+PkVZrXMvm1mTUi6UmbjS74FqIbX1a/klC6ntH1Yh9UZJIB0P2x4XrL/Y3uYjHBxSg2Q83YUqJQp3eSUzAzVBWGdft8Bn5Bs55p/HnMDWPxbfcvAObinPKT/jPlxLGY5VeKzSOYZBfb4iL/Jzq1zt6ZxbRo2rsiwjciuvWaYAbkLoYF7Gp4X9P1+AiRZFM6xzT/nx+H03piusyuhfn1OvhHtX8KwGHThgRLNyYlSmhfZCvYYALo9/sj8ExhV3z/hxVD6xX9+f6KNxz7i/gKo5n47olo77ubUm+JeEG2BLhCXMjau8gzMlLSlsvFpq4Fv+TuHnAZ+Jbcf67yjinXgPXNxuLSMtAIJcSPpdeiOaA0A24BNuGWtePsO8phye8gzoa1098Jq5iEsm4iz/hQsWpOKc8EheS/xnXaozkmfQBuFHgofkDnsc9IRJ6vPIl3MC9cAeSBiMC9+D6ltNxreKN/DE8H3Z3D3HozydfdwMuXD4O5+Qb4yqfj+G6PYqdKAhXmWnkHfC/A+l7x5PgQu3FznTHvijKg/h3M3idscALfn1/XHfUlkiuIVvK1pJwA2yJ8IS5/rnXS+o8460XraZ3Xotwoe3a/oY8KXRDDuQ7hggm4QljvzErfz7dSAbUFdRP/jhwgU+7Htfl84Z3zBFPPuMd8yr2zTl+EG5AaJMSaPXHPUY4CTeqv2rgu+GEOaqdwiMCQwPHJQf4ayTHlTj057NvMO4yfD924LtOuFb7DQG7H6H4mS/b4CpLJ+Mqt9n4ip3/PhS5OgkXTi+2uwPXhXWOP2734ip3KbjBdaER/dVj9fuxJTmXhBtgSwlOWoydUTycWwmde01cy2N0vrS/4gaUXYVr1fXFzbZW4meUk22h6H7yO3BdCKHZz2pSwomC/Pahl/NENYjUax3v7Q4NgCxp1KawiECozG1CNofj3P02Me/PDzjb3rgBtBfl+74X7umJTsH8Reg1wUVPzgukHY176VNwzoBMf3yODcfOwHatcE9XvIarFD5OvhkmbSm/i70P3kgaRCQ0KPEKde8PT1XVXBE5EDcD19m4EL3iBpdNFxHRcnIRi8ixuEeZrsRNPnIQLoSai2s1vgv8U1V3xGBfJ+EqDplagnfQ59MKjcDvo6o/RKkzER+ZwbU0d4hIiqruFpEqqrorTK0/4Z4fX4GbX/6/uC6Ojbhw/Lm4SkNv3JztVxRne0Dzc6+1Cdd1MllVJwfyPQ68pKrvFncL27vxAAAQBklEQVR9isi5uGfRrxSRSrjz3g0X5r8BNyBuP1zo/jpVfT2c8ufbR2VVzRORe3DjDlrjoiubI9Uyyhbm4I2kQURq4R4ruklVZ/q00M29Om5il2+APFXdWp6cewjvRCbiwr31cIPjzsA5oe+AI1T11xjtq7qqbomR1km4EfndAC3peRGR43Gt6kNV9QcR2T9UoQn3fItIP/a9p6Aezln+GfdUywLclK53ikglVd0jIlVVdXsEmnVxkaSJ7Hsd8j9VdYKInIWbFbOfqm4Iw9ajcRGqu3DzEVTFnftpuBdGTcZV8nJUdXFJrvngNiJSF3ff/28kGkbZxBy8kVSIyBj8W7tUdWmg9XEyrj/+CuC38ubYg4jIANw0wr1UdZOIHIRzTtVUdV1CjSuCWFUYookIiEht3NvPTvIRnoZe69+qOlVEWuIiIc+o6l/8NsW1svNrNsI9qvaSqr4qIp1wYwbm454gOFdVV4RpbzXcXP+jcI+9TcCF7Jv49CvCjVoUs59yVxk2iicl0QYYRj5exbW4LhKRl4F3ROQIXCvn2li1XpMZVZ0pInuAj0XkMFX9OdE2hUOsogGqOltEUoE5IhJRRMBXiAYB94vIe6q6QUR2AbV9ZXG1iBwHvC4iDwKbitMuQPNbEdkN1PVRgM9FpBduAF5lVf0pgrJuAx4RkWdVdVMo3VcaOgJ1ROS7aJ2zOfeKibXgjaRDROrhJge6BPfYUnNgnKq+VpFaIrHsJy+LRBMRiGV/fhGaZ6nq9lCovyR2FrCPKrgR72OBv4S6qgyjJJiDN5IW7+j34KYQza5Izj1ELPvJKxqx6M+PRDMG9lbBjbW4E5igqtNjoWtUXMzBG4ZRbonVCP94awa0q+AeX/y+IlZojdhiffCGYZRbounPL03NgPYu3MQ81m9uRI214A3DKPfEo6vDuk+MZMccvGEYhmGUQyol2gDDMAzDMGKPOXjDMAzDKIeYgzcMwzCMcog5eMMwDMMoh5iDN4wIEJE8EVkqIl+KyFQ/l3hJtY4RkRn+82ARuamIvLVE5JIS7OMOEbku3PR8eZ4WkdMi2FcTEfkyUhsNw4gP5uANIzK2q2pnVW2Pe43rRcEvxRHx70pVX1fVcUVkqYWbutcwDCMszMEbRsmZD7TwLdeVIvIs7k1gDUXkBBH5SESW+JZ+dXCvHRWRr0VkCTAkJCQio0Rkkv9cT0ReFZHP/XI4MA5o7qMH432+60XkUxFZJiJ3BrRuFpFVIvIB7t3fRSIiF3idz0XkP/miEseLyCKvN9Dnrywi4wP7vjDaA2kYRuwxB28YJUBEUoD+wBc+qSXwmKpmAFuBW4DjVbUrsAi4RkT2B54EBgGZwKGFyE8E3lPVTkBXYDlwE7DGRw+uF5ET/D57AJ2BTBE5SkQyce+P7wycCHQPozivqGp3v7+vgPMC3zXx+xgAPO7LcB7wq6p29/oXiEjTMPZjGEYpYlPVGkZkVBWRpf7zfOCfuDeLrVfVj316L6AdsEBEAFKBj4A2wFpVXQ0gIs/j3vmdn2OBEQCqmgf8Ku6d8EFO8Mtnfr06zuHXAF71ryFFRF4Po0ztReQeXDdAddzb0kK87N+UtlpEvvFlOAHoGOifP9Dve1UY+zIMo5QwB28YkbFdVTsHE7wT3xpMAt5Q1eH58v1uuygRYKyqPpFvH1eVQOtp4GT/XvNRwDGB7/JPdal+35erarAigIg0KcG+DcOIExaiN4zY8zFwhIi0ABCRA0SkFfA10EREmvt8wwvZ/i3gYr9tZRE5ENiMa52HmAuMDvTtp4lIXeB94GQRqSoiNXDdAcVRA/jOv8nsrHzfDRWRSt7mZsBKv++LfX5EpJWIHBDGfgzDKEWsBW8YMUZVf/Qt4RdFZD+ffIuqrhKRMcBMEdmGC/HXKEDiSmCyiJwH5AEXq+pHIrLAP4Y22/fDtwU+8hGELcDZqrpERKYAnwM/AJ+GYfKtwCfAj/5/0KZvgYVATeAiVd0hIv/A9c0vEbfzH4GTwzs6hmGUFvayGcMwDMMoh1iI3jAMwzDKIebgDcMwDKMcYg7eSBgisp+ITBGRLBH5pLBR2CJypZ8adnlwlLjfdqlf1oUeXxORHoH0z0XkFJ++v4gs9GnLg5PDxKAs/xCRdhFuU+pTu4rIn/3xXikifQvJ09Sfjyx/jFN9+jUissJPbvOWiDQObNNIROaJyFc+TxOfPj9wLjaKyGth2HiF13lBRNqImzBopxQztW4yIW5Co5X+GBY4BbGINPbHcZmIvCsi6YHv5ojI/8RPZRxIL+zchPVbMioYqmqLLXsXIKUU93UJ8Lj/fAYwpYA87XGzw1XDDQp9E2hRQL4Hgdv852qhcgD1cYPNUnCPd1X36VVwA8p6JfBYNwG+LMX9tcMNvtsPaAqsASoXkO9l4Az/+XHcID+APkA1//ni4PkC3gX+5D9XD+XLp/sfYEQYdn4NpPvPdXGT6dwLXFeKx6rEvwOgsj+2zXBzIHwOtCsg31RgpP98LPBc4LvjcE9AzAjz3BT7W7Kl4i3Wgi8j/H975x5rV1HF4e/HG4MUaZsoRYFCFSrqpTYQalpebSwaEUsjYCiCIUqCYEhQMUQ0KMjDgEoxJQi9xYAQxFYilHd5CLaWvm6JaKxQ46WmvFqwwWKBn3+stdvpuec+KCTQ2/mSnZwze55r9t4zs2ZmjaQ5khblyPPrhftkhTnUZZIeSLfdJM2UtDxHByek+7oi3FRJnfm7U9IMSQuAy3ME/CdJSyQ9Lulj6W97ST/N0XSXpLMlHV2OyiRNkjR7gMX6IjArf/8WOCZXZZccBCyw/art14GHKUy8ZpoCvgz8BqDwC7ALuZfbQSODHfNyxnGRpONaM6g4lGVWjkT/KWmKpMtTtndr01axhySNTRl1poyWSzo37x8g6f6sp8XatFWuSWffTGNxXuPS/UOSHtGmA27G95bGAOV9i+3XbD8DrCCs1LXK8miiPiDq5/iU3zynAR1iK+DeGWY00SDel/7WFf6aeHfPeOfk/7GK1fit8p5BNIxzJZ1r+znbC4ENfRVM0hGFpmCJYosgkr6bMlom6dJ065A0P5/h2UojQlmHP5P0BPAtScMVpnsX5vWZ/gScHAqssP207f8BtxCyb2U08GD+nlf6sf0AsTWyLGOvdcPA3qXKNkbdJrf18DXbL0naFVgo6XZiiuU6YILtZyTtmX6/T5gS/QSAelpBa8fewDjbb+THeLzt1yVNBC4BTiCsru0LdOS9PYE1wC8lDbf9PHA6cEOmeyvtbaFfaftGYATwL4CM72VgKPBC4fdJ4GJJQ4H/EuZXn2iJbzyw2mkhLtM+LPOxDzCtafAlbQ8sAg4ArrG9INO/sA/Z7E+MXkcTFulOsP2d7Mh8nmy0kg5ghOMwGiTtke43AZfanq0w97odMTpteI4YAa+XNIrorIwFvgLcY/vizPv7ektD0rfpuY8d4BHb5xDynl+4d6dbyVBgbdFBaucHwlzt3Pz9UWCtpN8RmoH7gfMdVvgajgcesP0KgO0ngDNaI7V9pqTJwFG2X2i93wfnAWfZfkxhG2C9pGOJhu8w268W78eNhKGehyVdBPwAaKZ+drI9FkDSzcBVtv8o6SPE/v+DJB0FXNUmD6/aHkfxXCfdwGFt/C8jOqs/B74EvF/SUNsv9lLGvupmIO9SZRujNvBbD+co55KBDxOmQYcTH+9nAGy/lPcnEmo60n3NAOK/rfggDwFmZUNjYqTbxDuj+cA06Un6NXCKpJnA4Wwys3rilhS0xPZTki4D7iWsxS0l9oaXnEyO3otwC4CPK/aKz5I01/b6LGNHNoqzJR1su7958Lm2N0haTqhf70735USHp+RpYKSkq4E7gXtzNDnC9uzM23rYaAGvYUdgusLa3RtEowmxj/2G1BTMsb1UYTJ2szQy3iuAK/opy9tG0ilE5+OIdNqB6GQdQuybvxU4jTDj23Ay0GPE/g7yGHClpJsI2/rd2Tmd2WgTsoM8BNjD9sMZbhahKm+4tfg9ERhd1NPuknazPY/oZL1dziPq/DTCQNGz9Hy2K5UtpqrotwIkHUl8bA53HAiyhFA9v1VKowet4UtTqz8C5uUI8QsDSGsmcArxEb+tGC2Xi+DK69QM9yzRWWkObxkC9Bi92L7e9qdtTyA0Bhttnme4KWz+YS7DPkUYgTm4xX0toRad3E/ZAF7LMG8CG2w3cnyTlk5ydqY+RcxJn8nAG7VzgdUZdiwxd4vtR4AJhKw6JZ3aWxqK0+XayfsXmcZGeSd7p1vJi8AeKdcefrLRvAA4zvZr6dwNLE2V9OuERmNMEWYYoba+c4Cy6BdJZxXl28tx1O4ZwK7EGQAHbmHU5XuwHbFGoyOvEbbXSTqqFzk/nuEGImdsr7I9xfYhhEyb57I3+qqbAb1LlW2L2sBvHQwB1qSa8UDiMBMIdesE5UlehQryPuCsJnChol8t6SDFeeWNNqC39JoPx2mF+33AN5oPTJOe7VXAKuIEtZmNZ9snFh/H8roxvdwBfDV/TwUeLBrPjShMsJJq0inAzcXticBfbXcX/vcr8rgPcUDKypxTbdTZuwKTiAVdSPpJoSHZYrIx28727SmPMbb/A3RLOj797KzNj2SFkPm/sxMxjdAUNPlfbfs6oiEf0y4NiBF8L/I+J9O4Azgp09+P0AL9ucxEyn8eUR8Q9fP7zMshwLVE4/5cEWwh0fAMz/9HA38p7k8lFoutL+R0qOJ43S3C9jVF+VZJ2t/2ctuXZX4OJJ7X0xtZS9rT9svAGknjM6ppxLqOdtwLnF3kuSPTnteLnMcV8hiVz+FOhDatx6E/kobluwjwPXJqq48y91o3DPBdqmxj+D2w0q9efV/Eque5xFGec4iR25F571hiRL+MOOAEYhXzLGL+ehkwJd2nEqt75wPTgc507wSmFukdToySlwA/Blam+w7AlcTHexnwzSLMScD8t1iuXQj16AqioRmZ7nsBdxX+Hi3SPKYljk7ChGrpNo04YnUpsJg4SAXgk1mmrpTNhUWYPxAaktY8/pBi9Tawrt29rJOxxMh6caa9FDg2748iFlR1EWsARlKsos/7XVnGy5p0iI/2k5nvR4k57rZpDFDmF+Qz8LcyHHAXsFf+Hpn1sSLrZ+d0v5/QMjTp3lGEn5T5X551slNx7yFgcks+pgLX9pLHlcCw/P1BQkPwCrA2f+/eJszVKacuYrqmyfP5+ewsBS5Jtw7iHegi3qcPlHVYxDmM0Ax1ZRwz3oKcP0e8Q/8ALijcLyI6SI0M/p7+ftXkuXjmnyfWnXQDn+2nbtq+S/Xatq9qqrbyjiBpOrDE9vX9en4PIuke2233hVfeeSRdQWwL63q381KpDFZqA19520haRMxdTvKmedlKpVKpvIvUBr5SqVQqlUFIXWRXqVQqlcogpDbwlUqlUqkMQmoDX6lUKpXKIKQ28JVKpVKpDEJqA1+pVCqVyiCkNvCVSqVSqQxC/g+o9C0C13R9UwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 720x432 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "tags = sorted(set(y_true+y_pred))\n",
    "cnf_matrix = confusion_matrix(y_true, y_pred)\n",
    "plot_confusion_matrix(f1_score(y_true, y_pred, average='macro'), cnf_matrix, target_names=tags, title=name, normalize=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Note"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**##** label is comming from original-to-tokenized alignment code in function <font color=blue>convert_single_example(ex_index, example, label_list, max_seq_length, tokenizer, mode)</font>  in *bert_pos.py*. Following suggestions from [WordPiece tokenization](https://github.com/google-research/bert) part, for pos tagging task input sentences are converted to the following format:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# orig_tokens = [\"John\", \"Johanson\", \"'s\",  \"house\"]\n",
    "# labels      = [\"NNP\",  \"NNP\",      \"POS\", \"NN\"]\n",
    "\n",
    "# bert_tokens = ['[CLS]\", \"john\", \"johan\", \"##son\", \"'\",   \"s\",  \"house\", \"[SEP]\"]\n",
    "# bert_labels = ['[CLS]\", \"NNP\",  \"NNP\",   \"##\",    \"POS\", \"##\", \"NN\",    \"[SEP]\"]"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
